✨ add scripts to output usage stats 📈

packages/usage-stats/.gitignore

@@ -0,0 +1,2 @@
+.env
+cache

packages/usage-stats/dependents.ts

@@ -0,0 +1,79 @@
+import * as fs from "fs";
+import fetch from "node-fetch";
+import { load as CheerioLoad } from "cheerio";
+
+const getPackages = async (repo: string) => {
+  const pageText = await fetch(
+    `https://github.com/${repo}/network/dependents`
+  ).then((res) => res.text());
+  const $ = CheerioLoad(pageText);
+
+  return $("#dependents .select-menu-list a")
+    .toArray()
+    .map((el) => {
+      const name = $(el).text().trim();
+      const href = $(el).attr("href");
+      const u = new URL(href!, "http://example.com");
+
+      return { name, id: u.searchParams.get("package_id")! };
+    });
+};
+
+const getDependentByPackage = async (repo: string, packageId: string) => {
+  const repos = [] as string[];
+
+  const pages = [];
+
+  let url:
+    | string
+    | null = `https://github.com/${repo}/network/dependents?package_id=${packageId}`;
+
+  while (url) {
+    console.log(url, repos.length);
+
+    await wait(1000 + Math.floor(Math.random() * 500));
+
+    const $ = CheerioLoad(await fetch(url).then((res) => res.text()));
+
+    const rs = $(`#dependents [data-hovercard-type="repository"]`)
+      .toArray()
+      .map((el) => $(el).attr("href")!.slice(1));
+
+    repos.push(...rs);
+
+    const nextButton = $(`#dependents a`)
+      .filter((_, el) => $(el).text().trim().toLowerCase() === "next")
+      .eq(0);
+
+    const href = nextButton ? nextButton.attr("href") : null;
+
+    pages.push({ url, rs, next: href });
+    fs.writeFileSync(
+      __dirname + `/out-${packageId}.json`,
+      JSON.stringify(pages)
+    );
+
+    url = href ? new URL(href, "https://github.com").toString() : null;
+  }
+
+  return repos;
+};
+
+export const getDependents = async (repo: string) => {
+  const packages = await getPackages(repo);
+
+  const ps: (typeof packages[number] & { dependents: string[] })[] = [];
+
+  for (const p of packages)
+    ps.push({ ...p, dependents: await getDependentByPackage(repo, p.id) });
+
+  return ps;
+};
+
+const wait = (delay = 0) => new Promise((r) => setTimeout(r, delay));
+
+(async () => {
+  const res = await getDependents("platane/snk");
+
+  fs.writeFileSync(__dirname + "/cache/out.json", JSON.stringify(res));
+})();

packages/usage-stats/getRunInfo.ts

@@ -0,0 +1,125 @@
+import * as fs from "fs";
+import fetch from "node-fetch";
+import { Octokit } from "octokit";
+
+require("dotenv").config();
+
+// @ts-ignore
+import packages from "./out.json";
+
+const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
+
+const getLastRunInfo = async (repo_: string) => {
+  const [owner, repo] = repo_.split("/");
+
+  try {
+    const {
+      data: { workflow_runs },
+    } = await octokit.request(
+      "GET /repos/{owner}/{repo}/actions/runs{?actor,branch,event,status,per_page,page,created,exclude_pull_requests,check_suite_id,head_sha}",
+      { owner, repo }
+    );
+
+    for (const r of workflow_runs) {
+      const { run_started_at, head_sha, path, conclusion } = r as {
+        run_started_at: string;
+        head_sha: string;
+        path: string;
+        conclusion: "failure" | "success";
+      };
+
+      const workflow_url = `https://raw.githubusercontent.com/${owner}/${repo}/${head_sha}/${path}`;
+
+      const workflow_file = await fetch(workflow_url).then((res) => res.text());
+
+      const [_, dependency, __, version] =
+        workflow_file.match(/uses\s*:\s*(Platane\/snk(\/svg-only)?@(\w*))/) ??
+        [];
+
+      const cronMatch = workflow_file.match(/cron\s*:([^\n]*)/);
+
+      if (dependency)
+        return {
+          dependency,
+          version,
+          run_started_at,
+          conclusion,
+          cron: cronMatch?.[1].replace(/["|']/g, "").trim(),
+          workflow_file,
+          workflow_url,
+        };
+    }
+  } catch (err) {
+    console.error(err);
+  }
+};
+
+const wait = (delay = 0) => new Promise((r) => setTimeout(r, delay));
+
+const getRepos = () => {
+  try {
+    return JSON.parse(fs.readFileSync(__dirname + "/cache/out.json").toString())
+      .map((p: any) => p.dependents)
+      .flat() as string[];
+  } catch (err) {
+    return [];
+  }
+};
+
+const getReposInfo = () => {
+  try {
+    return JSON.parse(
+      fs.readFileSync(__dirname + "/cache/stats.json").toString()
+    ) as any[];
+  } catch (err) {
+    return [];
+  }
+};
+const saveRepoInfo = (rr: any[]) => {
+  fs.writeFileSync(__dirname + "/cache/stats.json", JSON.stringify(rr));
+};
+
+(async () => {
+  const repos = getRepos();
+  const total = repos.length;
+
+  const reposInfo = getReposInfo().slice(0, -20);
+  for (const { repo } of reposInfo) {
+    const i = repos.indexOf(repo);
+    if (i >= 0) repos.splice(i, 1);
+  }
+
+  while (repos.length) {
+    const {
+      data: { rate },
+    } = await octokit.request("GET /rate_limit", {});
+
+    console.log(rate);
+    if (rate.remaining < 100) {
+      const delay = rate.reset - Math.floor(Date.now() / 1000);
+      console.log(
+        `waiting ${delay} second (${(delay / 60).toFixed(
+          1
+        )} minutes) for reset `
+      );
+      await wait(Math.max(0, delay) * 1000);
+    }
+
+    const rs = repos.splice(0, 20);
+
+    await Promise.all(
+      rs.map(async (repo) => {
+        reposInfo.push({ repo, ...(await getLastRunInfo(repo)) });
+
+        saveRepoInfo(reposInfo);
+
+        console.log(
+          reposInfo.length.toString().padStart(5, " "),
+          "/",
+          total,
+          repo
+        );
+      })
+    );
+  }
+})();

packages/usage-stats/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "@snk/usage-stats",
+  "version": "1.0.0",
+  "dependencies": {},
+  "devDependencies": {
+    "sucrase": "3.29.0",
+    "cheerio": "1.0.0-rc.12",
+    "node-fetch": "2.6.7",
+    "octokit": "2.0.11",
+    "dotenv": "16.0.3"
+  },
+  "scripts": {
+    "start": "sucrase-node stats.ts"
+  }
+}

packages/usage-stats/stats.ts

@@ -0,0 +1,59 @@
+import * as fs from "fs";
+
+type R = { repo: string } & Partial<{
+  dependency: string;
+  version: string;
+  run_started_at: string;
+  conclusion: "failure" | "success";
+  cron?: string;
+  workflow_file: string;
+}>;
+
+(async () => {
+  const repos: R[] = JSON.parse(
+    fs.readFileSync(__dirname + "/cache/stats.json").toString()
+  );
+
+  const total = repos.length;
+
+  const recent_repos = repos.filter(
+    (r) =>
+      new Date(r.run_started_at!).getTime() >
+      Date.now() - 7 * 24 * 60 * 60 * 1000
+  );
+
+  const recent_successful_repos = recent_repos.filter(
+    (r) => r?.conclusion === "success"
+  );
+
+  const versions = new Map();
+  for (const { dependency } of recent_successful_repos) {
+    versions.set(dependency, (versions.get(dependency) ?? 0) + 1);
+  }
+
+  console.log(`total ${total}`);
+  console.log(
+    `recent_repos ${recent_repos.length} (${(
+      (recent_repos.length / total) *
+      100
+    ).toFixed(2)}%)`
+  );
+  console.log(
+    `recent_successful_repos ${recent_successful_repos.length} (${(
+      (recent_successful_repos.length / total) *
+      100
+    ).toFixed(2)}%)`
+  );
+  console.log("versions");
+  for (const [name, count] of Array.from(versions.entries()).sort(
+    ([, a], [, b]) => b - a
+  ))
+    console.log(
+      `${(name as string).split("Platane/")[1].padEnd(20, " ")}  ${(
+        (count / recent_successful_repos.length) *
+        100
+      )
+        .toFixed(2)
+        .padStart(6, " ")}% ${count} `
+    );
+})();