🚀 refactor getgithubcontribution

packages/action/generateContributionSnake.ts

@@ -13,7 +13,7 @@ export const generateContributionSnake = async (
   console.log("🎣 fetching github user contribution");
   const { cells, colorScheme } = await getGithubUserContribution(userName);
 
-  const grid = userContributionToGrid(cells);
+  const grid = userContributionToGrid(cells, colorScheme);
   const snake = snake4;
 
   const drawOptions = {

packages/action/userContributionToGrid.ts

@@ -1,13 +1,20 @@
-import { setColor, createEmptyGrid } from "@snk/types/grid";
+import { setColor, createEmptyGrid, setColorEmpty } from "@snk/types/grid";
 import type { Cell } from "@snk/github-user-contribution";
 import type { Color } from "@snk/types/grid";
 
-export const userContributionToGrid = (cells: Cell[]) => {
+export const userContributionToGrid = (
+  cells: Cell[],
+  colorScheme: string[]
+) => {
   const width = Math.max(0, ...cells.map((c) => c.x)) + 1;
   const height = Math.max(0, ...cells.map((c) => c.y)) + 1;
 
   const grid = createEmptyGrid(width, height);
-  for (const c of cells) if (c.k) setColor(grid, c.x, c.y, c.k as Color);
+  for (const c of cells) {
+    const k = colorScheme.indexOf(c.color);
+    if (k > 0) setColor(grid, c.x, c.y, k as Color);
+    else setColorEmpty(grid, c.x, c.y);
+  }
 
   return grid;
 };

packages/demo/demo.interactive.ts

@@ -205,7 +205,7 @@ const onSubmit = async (userName: string) => {
     cells,
   };
 
-  const grid = userContributionToGrid(cells);
+  const grid = userContributionToGrid(cells, colorScheme);
   const chain = getBestRoute(grid, snake)!;
   chain.push(...getPathToPose(chain.slice(-1)[0], snake)!);
   dispose();

packages/github-user-contribution/README.md

@@ -2,6 +2,28 @@
 
 Get the github user contribution grid
 
+## Usage
+
+```js
+const { cells, colorScheme } = await getGithubUserContribution("platane");
+
+// colorScheme = [
+//    "#ebedf0",
+//    "#9be9a8",
+//    ...
+// ]
+// cells = [
+//    {
+//      x: 3,
+//      y: 0,
+//      count: 3,
+//      color: '#ebedf0',
+//      date:'2019-01-18'
+//    },
+//    ...
+// ]
+```
+
 ## Implementation
 
 Based on the html page. Which is very unstable. Might switch to using github api but afaik it's a bit complex.

packages/github-user-contribution/__tests__/formatParams.spec.ts

@@ -0,0 +1,18 @@
+import { formatParams } from "../formatParams";
+
+[
+  //
+  [{}, ""],
+  [{ year: 2017 }, "from=2017-01-01&to=2017-12-31"],
+  [{ from: new Date("2017-12-03") }, "from=2017-12-03"],
+  [{ from: "2017-12-03" }, "from=2017-12-03"],
+  [{ to: "2017-12-03" }, "to=2017-12-03"],
+].forEach(([params, res]) =>
+  it(`should format ${JSON.stringify(params)}`, () => {
+    expect(formatParams(params)).toBe(res);
+  })
+);
+
+it("should fail if the date is in the future", () => {
+  expect(() => formatParams({ to: new Date() })).toThrow(Error);
+});

packages/github-user-contribution/__tests__/getGithubUserContribution.spec.ts

@@ -1,14 +1,54 @@
 import { getGithubUserContribution } from "..";
 
-it("should get user contribution", async () => {
+describe("getGithubUserContribution", () => {
-  const { cells, colorScheme } = await getGithubUserContribution("platane");
+  const promise = getGithubUserContribution("platane");
 
-  expect(cells.length).toBeGreaterThan(300);
+  it("should resolve", async () => {
-  expect(colorScheme).toEqual([
+    await promise;
-    "#ebedf0",
+  });
-    "#9be9a8",
+
-    "#40c463",
+  it("should get colorScheme", async () => {
-    "#30a14e",
+    const { colorScheme } = await promise;
-    "#216e39",
+
-  ]);
+    expect(colorScheme).toEqual([
+      "#ebedf0",
+      "#9be9a8",
+      "#40c463",
+      "#30a14e",
+      "#216e39",
+    ]);
+  });
+
+  it("should get around 365 cells", async () => {
+    const { cells } = await promise;
+
+    expect(cells.length).toBeGreaterThan(340);
+    expect(cells.length).toBeLessThanOrEqual(367);
+  });
+
+  it("cells should have x / y coords representing to a 7 x (365/7) (minus unfilled last row)", async () => {
+    const { cells, colorScheme } = await promise;
+
+    expect(cells.length).toBeGreaterThan(300);
+    expect(colorScheme).toEqual([
+      "#ebedf0",
+      "#9be9a8",
+      "#40c463",
+      "#30a14e",
+      "#216e39",
+    ]);
+
+    const undefinedDays = Array.from({ length: Math.floor(365 / 7) - 1 })
+      .map((x) => Array.from({ length: 7 }).map((y) => ({ x, y })))
+      .flat()
+      .filter(({ x, y }) => cells.some((c) => c.x === x && c.y === y));
+
+    expect(undefinedDays).toEqual([]);
+  });
+});
+
+it("should match snapshot for year=2019", async () => {
+  expect(
+    await getGithubUserContribution("platane", { year: 2019 })
+  ).toMatchSnapshot();
 });

packages/github-user-contribution/formatParams.ts

@@ -0,0 +1,50 @@
+export type Options =
+  | { from?: string | Date; to?: string | Date }
+  | { year: number };
+
+export const formatParams = (options: Options = {}) => {
+  const sp = new URLSearchParams();
+
+  const o: any = { ...options };
+
+  if ("year" in options) {
+    const from = new Date();
+    from.setFullYear(options.year);
+    from.setMonth(0);
+    from.setDate(1);
+
+    const to = new Date();
+    to.setFullYear(options.year);
+    to.setMonth(11);
+    to.setDate(31);
+
+    o.from = from;
+    o.to = to;
+  }
+
+  for (const s of ["from", "to"])
+    if (o[s]) {
+      const value = formatDate(o[s]);
+
+      if (value >= formatDate(new Date()))
+        throw new Error("cannot get contribution for date in the future");
+
+      sp.set(s, value);
+    }
+
+  return sp.toString();
+};
+
+const formatDate = (input: Date | string) => {
+  const d = new Date(input);
+
+  const year = d.getFullYear();
+  const month = d.getMonth() + 1;
+  const date = d.getDate();
+
+  return [
+    year,
+    month.toString().padStart(2, "0"),
+    date.toString().padStart(2, "0"),
+  ].join("-");
+};

packages/github-user-contribution/index.ts

@@ -1,19 +1,34 @@
 import fetch from "node-fetch";
-import * as parser from "fast-xml-parser";
+import cheerio from "cheerio";
+import { formatParams, Options } from "./formatParams";
 
-const findNode = (o: any, condition: (x: any) => boolean): any => {
+/**
-  if (o && typeof o === "object") {
+ * get the contribution grid from a github user page
-    if (condition(o)) return o;
+ *
+ * @param userName github user name
+ * @param options set the time range: from / to or year
+ */
+export const getGithubUserContribution = async (
+  userName: string,
+  options: Options = {}
+) => {
+  // either use github.com/users/xxxx/contributions  for previous years
+  // or github.com/xxxx ( which gives the latest update to today result )
+  const url =
+    "year" in options || "from" in options || "to" in options
+      ? `https://github.com/users/${userName}/contributions?` +
+        formatParams(options)
+      : `https://github.com/${userName}`;
 
-    for (const c of Object.values(o)) {
+  const res = await fetch(url);
-      const res = findNode(c, condition);
+
-      if (res) return res;
+  if (!res.ok) throw new Error(res.statusText);
-    }
+
-  }
+  const resText = await res.text();
+
+  return parseUserPage(resText);
 };
 
-const ensureArray = (x: any) => (Array.isArray(x) ? x : [x]);
-
 const defaultColorScheme = [
   "#ebedf0",
   "#9be9a8",
@@ -23,11 +38,7 @@ const defaultColorScheme = [
 ];
 
 const parseUserPage = (content: string) => {
-  const o = parser.parse(content, {
+  const $ = cheerio.load(content);
-    attrNodeName: "attr",
-    attributeNamePrefix: "",
-    ignoreAttributes: false,
-  });
 
   //
   // parse colorScheme
@@ -35,57 +46,84 @@ const parseUserPage = (content: string) => {
   const colorSchemeMap: Record<string, number> = Object.fromEntries(
     defaultColorScheme.map((color, i) => [color, i])
   );
-  const legend = findNode(
+  $("ul.legend > li")
-    o,
+    .toArray()
-    (x) => x.attr && x.attr.class && x.attr.class.trim() === "legend"
+    .forEach((x, i) => {
-  );
+      const bgColor = x.attribs.style.match(/background\-color: +(.+)/)![1]!;
-  legend.li.forEach((x: any, i: number) => {
+      if (bgColor) {
-    const bgColor = x.attr.style.match(/background\-color: +(.+)/)![1]!;
+        const color = bgColor.replace(/\s/g, "");
-    if (bgColor) {
+        colorSchemeMap[color] = i;
-      const color = bgColor.replace(/\s/g, "");
-      colorSchemeMap[color] = i;
 
-      if (!color.startsWith("var(--")) colorScheme[i] = color;
+        if (!color.startsWith("var(--")) colorScheme[i] = color;
-    }
+      }
-  });
+    });
 
   //
   // parse cells
-  const svg = findNode(
+  const rawCells = $(".js-calendar-graph rect[data-count]")
-    o,
+    .toArray()
-    (x) =>
+    .map((x) => {
-      x.attr && x.attr.class && x.attr.class.trim() === "js-calendar-graph-svg"
+      const color = x.attribs.fill.trim();
-  );
+      const count = +x.attribs["data-count"];
+      const date = x.attribs["data-date"];
 
-  const cells = svg.g.g
+      const colorIndex = colorSchemeMap[color];
-    .map((g: any, x: number) =>
-      ensureArray(g.rect).map(({ attr }: any, y: number) => {
-        const color = attr.fill.trim();
-        const count = +attr["data-count"];
-        const date = attr["data-date"];
 
-        const k = colorSchemeMap[color];
+      if (colorIndex === -1) throw new Error("could not map the cell color");
 
-        if (k === -1) throw new Error("could not map the cell color");
+      return {
+        svgPosition: getSvgPosition(x),
+        color: colorScheme[colorIndex],
+        count,
+        date,
+      };
+    });
 
-        return { x, y, color, count, date, k };
+  const xMap: Record<number, true> = {};
-      })
+  const yMap: Record<number, true> = {};
-    )
+  rawCells.forEach(({ svgPosition: { x, y } }) => {
-    .flat();
+    xMap[x] = true;
+    yMap[y] = true;
+  });
+
+  const xRange = Object.keys(xMap)
+    .map((x) => +x)
+    .sort((a, b) => +a - +b);
+  const yRange = Object.keys(yMap)
+    .map((x) => +x)
+    .sort((a, b) => +a - +b);
+
+  const cells = rawCells.map(({ svgPosition, ...c }) => ({
+    ...c,
+    x: xRange.indexOf(svgPosition.x),
+    y: yRange.indexOf(svgPosition.y),
+  }));
 
   return { cells, colorScheme };
 };
 
-/**
+// returns the position of the svg elements, accounting for it's transform and it's parent transform
- * get the contribution grid from a github user page
+// ( only accounts for translate transform )
- *
+const getSvgPosition = (e: cheerio.Element): { x: number; y: number } => {
- * @param userName
+  if (!e || e.tagName === "svg") return { x: 0, y: 0 };
- */
-export const getGithubUserContribution = async (userName: string) => {
-  const res = await fetch(`https://github.com/${userName}`);
-  const resText = await res.text();
 
-  return parseUserPage(resText);
+  const p = getSvgPosition(e.parent);
+
+  if (e.attribs.x) p.x += +e.attribs.x;
+  if (e.attribs.y) p.y += +e.attribs.y;
+
+  if (e.attribs.transform) {
+    const m = e.attribs.transform.match(
+      /translate\( *([\.\d]+) *, *([\.\d]+) *\)/
+    );
+
+    if (m) {
+      p.x += +m[1];
+      p.y += +m[2];
+    }
+  }
+
+  return p;
 };
 
 type ThenArg<T> = T extends PromiseLike<infer U> ? U : T;

packages/github-user-contribution/package.json

@@ -2,10 +2,11 @@
   "name": "@snk/github-user-contribution",
   "version": "1.0.0",
   "dependencies": {
-    "node-fetch": "2.6.1",
+    "cheerio": "~1.0.0-rc.3",
-    "fast-xml-parser": "3.17.4"
+    "node-fetch": "2.6.1"
   },
   "devDependencies": {
+    "@types/cheerio": "0.22.22",
     "@types/node-fetch": "2.5.7"
   }
 }

yarn.lock

@@ -576,6 +576,13 @@
     "@types/connect" "*"
     "@types/node" "*"
 
+"@types/cheerio@0.22.22":
+  version "0.22.22"
+  resolved "https://registry.yarnpkg.com/@types/cheerio/-/cheerio-0.22.22.tgz#ae71cf4ca59b8bbaf34c99af7a5d6c8894988f5f"
+  integrity sha512-05DYX4zU96IBfZFY+t3Mh88nlwSMtmmzSYaQkKN48T495VV1dkHSah6qYyDTN5ngaS0i0VonH37m+RuzSM0YiA==
+  dependencies:
+    "@types/node" "*"
+
 "@types/color-name@^1.1.1":
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0"
@@ -1645,6 +1652,18 @@ char-regex@^1.0.2:
   resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf"
   integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==
 
+cheerio@~1.0.0-rc.3:
+  version "1.0.0-rc.3"
+  resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.3.tgz#094636d425b2e9c0f4eb91a46c05630c9a1a8bf6"
+  integrity sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==
+  dependencies:
+    css-select "~1.2.0"
+    dom-serializer "~0.1.1"
+    entities "~1.1.1"
+    htmlparser2 "^3.9.1"
+    lodash "^4.15.0"
+    parse5 "^3.0.1"
+
 chokidar@^2.1.8:
   version "2.1.8"
   resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917"
@@ -1904,7 +1923,7 @@ cross-spawn@^7.0.0:
     shebang-command "^2.0.0"
     which "^2.0.1"
 
-css-select@^1.1.0:
+css-select@^1.1.0, css-select@~1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/css-select/-/css-select-1.2.0.tgz#2b3a110539c5355f1cd8d314623e870b121ec858"
   integrity sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=
@@ -1915,11 +1934,11 @@ css-select@^1.1.0:
     nth-check "~1.0.1"
 
 css-tree@^1.0.0:
-  version "1.0.0"
+  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.0.0.tgz#21993fa270d742642a90409a2c0cb3ac0298adf6"
+  resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.1.2.tgz#9ae393b5dafd7dae8a622475caec78d3d8fbd7b5"
-  integrity sha512-CdVYz/Yuqw0VdKhXPBIgi8DO3NicJVYZNWeX9XcIuSp9ZoFT5IcleVRW07O5rMjdcx1mb+MEJPknTTEW7DdsYw==
+  integrity sha512-wCoWush5Aeo48GLhfHPbmvZs59Z+M7k5+B1xDnXbdWNcEF423DoFdqSWE0PM5aNk5nI5cp1q7ms36zGApY/sKQ==
   dependencies:
-    mdn-data "2.0.12"
+    mdn-data "2.0.14"
     source-map "^0.6.1"
 
 css-what@2.1:
@@ -2182,7 +2201,15 @@ dom-serializer@0:
     domelementtype "^2.0.1"
     entities "^2.0.0"
 
-domelementtype@1, domelementtype@^1.3.1:
+dom-serializer@~0.1.1:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0"
+  integrity sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==
+  dependencies:
+    domelementtype "^1.3.0"
+    entities "^1.1.1"
+
+domelementtype@1, domelementtype@^1.3.0, domelementtype@^1.3.1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f"
   integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==
@@ -2304,7 +2331,7 @@ enquirer@^2.3.4:
   dependencies:
     ansi-colors "^4.1.1"
 
-entities@^1.1.1:
+entities@^1.1.1, entities@~1.1.1:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56"
   integrity sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==
@@ -2639,11 +2666,6 @@ fast-levenshtein@~2.0.6:
   resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917"
   integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=
 
-fast-xml-parser@3.17.4:
-  version "3.17.4"
-  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.17.4.tgz#d668495fb3e4bbcf7970f3c24ac0019d82e76477"
-  integrity sha512-qudnQuyYBgnvzf5Lj/yxMcf4L9NcVWihXJg7CiU1L+oUCq8MUnFEfH2/nXR/W5uq+yvUN1h7z6s7vs2v1WkL1A==
-
 faye-websocket@^0.10.0:
   version "0.10.0"
   resolved "https://registry.yarnpkg.com/faye-websocket/-/faye-websocket-0.10.0.tgz#4e492f8d04dfb6f89003507f6edbf2d501e7c6f4"
@@ -3049,7 +3071,7 @@ html-webpack-plugin@5.0.0-alpha.10:
     pretty-error "^2.1.1"
     tapable "2.0.0"
 
-htmlparser2@^3.3.0:
+htmlparser2@^3.3.0, htmlparser2@^3.9.1:
   version "3.10.1"
   resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f"
   integrity sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==
@@ -4161,7 +4183,7 @@ lodash.sortby@^4.7.0:
   resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
   integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=
 
-lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.20:
+lodash@^4.15.0, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.20:
   version "4.17.20"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52"
   integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==
@@ -4214,10 +4236,10 @@ map-visit@^1.0.0:
   dependencies:
     object-visit "^1.0.0"
 
-mdn-data@2.0.12:
+mdn-data@2.0.14:
-  version "2.0.12"
+  version "2.0.14"
-  resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.12.tgz#bbb658d08b38f574bbb88f7b83703defdcc46844"
+  resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.14.tgz#7113fc4281917d63ce29b43446f701e68c25ba50"
-  integrity sha512-ULbAlgzVb8IqZ0Hsxm6hHSlQl3Jckst2YEQS7fODu9ilNWy2LvcoSY7TRFIktABP2mdppBioc66va90T+NUs8Q==
+  integrity sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==
 
 media-typer@0.3.0:
   version "0.3.0"
@@ -4827,6 +4849,13 @@ parse5@5.1.1:
   resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178"
   integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==
 
+parse5@^3.0.1:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/parse5/-/parse5-3.0.3.tgz#042f792ffdd36851551cf4e9e066b3874ab45b5c"
+  integrity sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==
+  dependencies:
+    "@types/node" "*"
+
 parseurl@~1.3.2, parseurl@~1.3.3:
   version "1.3.3"
   resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4"