80 lines
2.1 KiB
TypeScript
80 lines
2.1 KiB
TypeScript
import * as fs from "fs";
|
|
import fetch from "node-fetch";
|
|
import { load as CheerioLoad } from "cheerio";
|
|
|
|
const getPackages = async (repo: string) => {
|
|
const pageText = await fetch(
|
|
`https://github.com/${repo}/network/dependents`
|
|
).then((res) => res.text());
|
|
const $ = CheerioLoad(pageText);
|
|
|
|
return $("#dependents .select-menu-list a")
|
|
.toArray()
|
|
.map((el) => {
|
|
const name = $(el).text().trim();
|
|
const href = $(el).attr("href");
|
|
const u = new URL(href!, "http://example.com");
|
|
|
|
return { name, id: u.searchParams.get("package_id")! };
|
|
});
|
|
};
|
|
|
|
const getDependentByPackage = async (repo: string, packageId: string) => {
|
|
const repos = [] as string[];
|
|
|
|
const pages = [];
|
|
|
|
let url:
|
|
| string
|
|
| null = `https://github.com/${repo}/network/dependents?package_id=${packageId}`;
|
|
|
|
while (url) {
|
|
console.log(url, repos.length);
|
|
|
|
await wait(1000 + Math.floor(Math.random() * 500));
|
|
|
|
const $ = CheerioLoad(await fetch(url).then((res) => res.text()));
|
|
|
|
const rs = $(`#dependents [data-hovercard-type="repository"]`)
|
|
.toArray()
|
|
.map((el) => $(el).attr("href")!.slice(1));
|
|
|
|
repos.push(...rs);
|
|
|
|
const nextButton = $(`#dependents a`)
|
|
.filter((_, el) => $(el).text().trim().toLowerCase() === "next")
|
|
.eq(0);
|
|
|
|
const href = nextButton ? nextButton.attr("href") : null;
|
|
|
|
pages.push({ url, rs, next: href });
|
|
fs.writeFileSync(
|
|
__dirname + `/out-${packageId}.json`,
|
|
JSON.stringify(pages)
|
|
);
|
|
|
|
url = href ? new URL(href, "https://github.com").toString() : null;
|
|
}
|
|
|
|
return repos;
|
|
};
|
|
|
|
export const getDependents = async (repo: string) => {
|
|
const packages = await getPackages(repo);
|
|
|
|
const ps: (typeof packages[number] & { dependents: string[] })[] = [];
|
|
|
|
for (const p of packages)
|
|
ps.push({ ...p, dependents: await getDependentByPackage(repo, p.id) });
|
|
|
|
return ps;
|
|
};
|
|
|
|
const wait = (delay = 0) => new Promise((r) => setTimeout(r, delay));
|
|
|
|
(async () => {
|
|
const res = await getDependents("platane/snk");
|
|
|
|
fs.writeFileSync(__dirname + "/cache/out.json", JSON.stringify(res));
|
|
})();
|