✨ add scripts to output usage stats 📈

packages/usage-stats/dependents.ts

@@ -0,0 +1,79 @@
+import * as fs from "fs";
+import fetch from "node-fetch";
+import { load as CheerioLoad } from "cheerio";
+
+const getPackages = async (repo: string) => {
+  const pageText = await fetch(
+    `https://github.com/${repo}/network/dependents`
+  ).then((res) => res.text());
+  const $ = CheerioLoad(pageText);
+
+  return $("#dependents .select-menu-list a")
+    .toArray()
+    .map((el) => {
+      const name = $(el).text().trim();
+      const href = $(el).attr("href");
+      const u = new URL(href!, "http://example.com");
+
+      return { name, id: u.searchParams.get("package_id")! };
+    });
+};
+
+const getDependentByPackage = async (repo: string, packageId: string) => {
+  const repos = [] as string[];
+
+  const pages = [];
+
+  let url:
+    | string
+    | null = `https://github.com/${repo}/network/dependents?package_id=${packageId}`;
+
+  while (url) {
+    console.log(url, repos.length);
+
+    await wait(1000 + Math.floor(Math.random() * 500));
+
+    const $ = CheerioLoad(await fetch(url).then((res) => res.text()));
+
+    const rs = $(`#dependents [data-hovercard-type="repository"]`)
+      .toArray()
+      .map((el) => $(el).attr("href")!.slice(1));
+
+    repos.push(...rs);
+
+    const nextButton = $(`#dependents a`)
+      .filter((_, el) => $(el).text().trim().toLowerCase() === "next")
+      .eq(0);
+
+    const href = nextButton ? nextButton.attr("href") : null;
+
+    pages.push({ url, rs, next: href });
+    fs.writeFileSync(
+      __dirname + `/out-${packageId}.json`,
+      JSON.stringify(pages)
+    );
+
+    url = href ? new URL(href, "https://github.com").toString() : null;
+  }
+
+  return repos;
+};
+
+export const getDependents = async (repo: string) => {
+  const packages = await getPackages(repo);
+
+  const ps: (typeof packages[number] & { dependents: string[] })[] = [];
+
+  for (const p of packages)
+    ps.push({ ...p, dependents: await getDependentByPackage(repo, p.id) });
+
+  return ps;
+};
+
+const wait = (delay = 0) => new Promise((r) => setTimeout(r, delay));
+
+(async () => {
+  const res = await getDependents("platane/snk");
+
+  fs.writeFileSync(__dirname + "/cache/out.json", JSON.stringify(res));
+})();