Bennett/snk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🚀 refactor getgithubcontribution

Browse Source
This commit is contained in:
platane
2020-11-30 10:56:18 +01:00
parent 57a7e7cf36
commit 485b70d30b
11 changed files with 2878 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/action/generateContributionSnake.ts
View File

@@ -13,7 +13,7 @@ export const generateContributionSnake = async (
console.log("🎣 fetching github user contribution");
const { cells, colorScheme } = await getGithubUserContribution(userName);
const grid = userContributionToGrid(cells);
const grid = userContributionToGrid(cells, colorScheme);
const snake = snake4;
const drawOptions = {

13
packages/action/userContributionToGrid.ts
View File

@@ -1,13 +1,20 @@
import { setColor, createEmptyGrid } from "@snk/types/grid";
import { setColor, createEmptyGrid, setColorEmpty } from "@snk/types/grid";
import type { Cell } from "@snk/github-user-contribution";
import type { Color } from "@snk/types/grid";
export const userContributionToGrid = (cells: Cell[]) => {
export const userContributionToGrid = (
cells: Cell[],
colorScheme: string[]
) => {
const width = Math.max(0, ...cells.map((c) => c.x)) + 1;
const height = Math.max(0, ...cells.map((c) => c.y)) + 1;
const grid = createEmptyGrid(width, height);
for (const c of cells) if (c.k) setColor(grid, c.x, c.y, c.k as Color);
for (const c of cells) {
const k = colorScheme.indexOf(c.color);
if (k > 0) setColor(grid, c.x, c.y, k as Color);
else setColorEmpty(grid, c.x, c.y);
}
return grid;
};

2
packages/demo/demo.interactive.ts
View File

@@ -205,7 +205,7 @@ const onSubmit = async (userName: string) => {
cells,
};
const grid = userContributionToGrid(cells);
const grid = userContributionToGrid(cells, colorScheme);
const chain = getBestRoute(grid, snake)!;
chain.push(...getPathToPose(chain.slice(-1)[0], snake)!);
dispose();

22
packages/github-user-contribution/README.md
View File

@@ -2,6 +2,28 @@
Get the github user contribution grid
## Usage
```js
const { cells, colorScheme } = await getGithubUserContribution("platane");
// colorScheme = [
// "#ebedf0",
// "#9be9a8",
// ...
// ]
// cells = [
// {
// x: 3,
// y: 0,
// count: 3,
// color: '#ebedf0',
// date:'2019-01-18'
// },
// ...
// ]
```
## Implementation
Based on the html page. Which is very unstable. Might switch to using github api but afaik it's a bit complex.

2584
packages/github-user-contribution/__tests__/__snapshots__/getGithubUserContribution.spec.ts.snap Normal file
View File

File diff suppressed because it is too large Load Diff

18
packages/github-user-contribution/__tests__/formatParams.spec.ts Normal file
View File

@@ -0,0 +1,18 @@
import { formatParams } from "../formatParams";
[
//
[{}, ""],
[{ year: 2017 }, "from=2017-01-01&to=2017-12-31"],
[{ from: new Date("2017-12-03") }, "from=2017-12-03"],
[{ from: "2017-12-03" }, "from=2017-12-03"],
[{ to: "2017-12-03" }, "to=2017-12-03"],
].forEach(([params, res]) =>
it(`should format ${JSON.stringify(params)}`, () => {
expect(formatParams(params)).toBe(res);
})
);
it("should fail if the date is in the future", () => {
expect(() => formatParams({ to: new Date() })).toThrow(Error);
});

60
packages/github-user-contribution/__tests__/getGithubUserContribution.spec.ts
View File

@@ -1,14 +1,54 @@
import { getGithubUserContribution } from "..";
it("should get user contribution", async () => {
const { cells, colorScheme } = await getGithubUserContribution("platane");
describe("getGithubUserContribution", () => {
const promise = getGithubUserContribution("platane");
expect(cells.length).toBeGreaterThan(300);
expect(colorScheme).toEqual([
"#ebedf0",
"#9be9a8",
"#40c463",
"#30a14e",
"#216e39",
]);
it("should resolve", async () => {
await promise;
});
it("should get colorScheme", async () => {
const { colorScheme } = await promise;
expect(colorScheme).toEqual([
"#ebedf0",
"#9be9a8",
"#40c463",
"#30a14e",
"#216e39",
]);
});
it("should get around 365 cells", async () => {
const { cells } = await promise;
expect(cells.length).toBeGreaterThan(340);
expect(cells.length).toBeLessThanOrEqual(367);
});
it("cells should have x / y coords representing to a 7 x (365/7) (minus unfilled last row)", async () => {
const { cells, colorScheme } = await promise;
expect(cells.length).toBeGreaterThan(300);
expect(colorScheme).toEqual([
"#ebedf0",
"#9be9a8",
"#40c463",
"#30a14e",
"#216e39",
]);
const undefinedDays = Array.from({ length: Math.floor(365 / 7) - 1 })
.map((x) => Array.from({ length: 7 }).map((y) => ({ x, y })))
.flat()
.filter(({ x, y }) => cells.some((c) => c.x === x && c.y === y));
expect(undefinedDays).toEqual([]);
});
});
it("should match snapshot for year=2019", async () => {
expect(
await getGithubUserContribution("platane", { year: 2019 })
).toMatchSnapshot();
});

50
packages/github-user-contribution/formatParams.ts Normal file
View File

@@ -0,0 +1,50 @@
export type Options =
| { from?: string | Date; to?: string | Date }
| { year: number };
export const formatParams = (options: Options = {}) => {
const sp = new URLSearchParams();
const o: any = { ...options };
if ("year" in options) {
const from = new Date();
from.setFullYear(options.year);
from.setMonth(0);
from.setDate(1);
const to = new Date();
to.setFullYear(options.year);
to.setMonth(11);
to.setDate(31);
o.from = from;
o.to = to;
}
for (const s of ["from", "to"])
if (o[s]) {
const value = formatDate(o[s]);
if (value >= formatDate(new Date()))
throw new Error("cannot get contribution for date in the future");
sp.set(s, value);
}
return sp.toString();
};
const formatDate = (input: Date | string) => {
const d = new Date(input);
const year = d.getFullYear();
const month = d.getMonth() + 1;
const date = d.getDate();
return [
year,
month.toString().padStart(2, "0"),
date.toString().padStart(2, "0"),
].join("-");
};

146
packages/github-user-contribution/index.ts
View File

@@ -1,19 +1,34 @@
import fetch from "node-fetch";
import * as parser from "fast-xml-parser";
import cheerio from "cheerio";
import { formatParams, Options } from "./formatParams";
const findNode = (o: any, condition: (x: any) => boolean): any => {
if (o && typeof o === "object") {
if (condition(o)) return o;
/**
* get the contribution grid from a github user page
*
* @param userName github user name
* @param options set the time range: from / to or year
*/
export const getGithubUserContribution = async (
userName: string,
options: Options = {}
) => {
// either use github.com/users/xxxx/contributions for previous years
// or github.com/xxxx ( which gives the latest update to today result )
const url =
"year" in options || "from" in options || "to" in options
? `https://github.com/users/${userName}/contributions?` +
formatParams(options)
: `https://github.com/${userName}`;
for (const c of Object.values(o)) {
const res = findNode(c, condition);
if (res) return res;
}
}
const res = await fetch(url);
if (!res.ok) throw new Error(res.statusText);
const resText = await res.text();
return parseUserPage(resText);
};
const ensureArray = (x: any) => (Array.isArray(x) ? x : [x]);
const defaultColorScheme = [
"#ebedf0",
"#9be9a8",
@@ -23,11 +38,7 @@ const defaultColorScheme = [
];
const parseUserPage = (content: string) => {
const o = parser.parse(content, {
attrNodeName: "attr",
attributeNamePrefix: "",
ignoreAttributes: false,
});
const $ = cheerio.load(content);
//
// parse colorScheme
@@ -35,57 +46,84 @@ const parseUserPage = (content: string) => {
const colorSchemeMap: Record<string, number> = Object.fromEntries(
defaultColorScheme.map((color, i) => [color, i])
);
const legend = findNode(
o,
(x) => x.attr && x.attr.class && x.attr.class.trim() === "legend"
);
legend.li.forEach((x: any, i: number) => {
const bgColor = x.attr.style.match(/background\-color: +(.+)/)![1]!;
if (bgColor) {
const color = bgColor.replace(/\s/g, "");
colorSchemeMap[color] = i;
$("ul.legend > li")
.toArray()
.forEach((x, i) => {
const bgColor = x.attribs.style.match(/background\-color: +(.+)/)![1]!;
if (bgColor) {
const color = bgColor.replace(/\s/g, "");
colorSchemeMap[color] = i;
if (!color.startsWith("var(--")) colorScheme[i] = color;
}
});
if (!color.startsWith("var(--")) colorScheme[i] = color;
}
});
//
// parse cells
const svg = findNode(
o,
(x) =>
x.attr && x.attr.class && x.attr.class.trim() === "js-calendar-graph-svg"
);
const rawCells = $(".js-calendar-graph rect[data-count]")
.toArray()
.map((x) => {
const color = x.attribs.fill.trim();
const count = +x.attribs["data-count"];
const date = x.attribs["data-date"];
const cells = svg.g.g
.map((g: any, x: number) =>
ensureArray(g.rect).map(({ attr }: any, y: number) => {
const color = attr.fill.trim();
const count = +attr["data-count"];
const date = attr["data-date"];
const colorIndex = colorSchemeMap[color];
const k = colorSchemeMap[color];
if (colorIndex === -1) throw new Error("could not map the cell color");
if (k === -1) throw new Error("could not map the cell color");
return {
svgPosition: getSvgPosition(x),
color: colorScheme[colorIndex],
count,
date,
};
});
return { x, y, color, count, date, k };
})
)
.flat();
const xMap: Record<number, true> = {};
const yMap: Record<number, true> = {};
rawCells.forEach(({ svgPosition: { x, y } }) => {
xMap[x] = true;
yMap[y] = true;
});
const xRange = Object.keys(xMap)
.map((x) => +x)
.sort((a, b) => +a - +b);
const yRange = Object.keys(yMap)
.map((x) => +x)
.sort((a, b) => +a - +b);
const cells = rawCells.map(({ svgPosition, ...c }) => ({
...c,
x: xRange.indexOf(svgPosition.x),
y: yRange.indexOf(svgPosition.y),
}));
return { cells, colorScheme };
};
/**
* get the contribution grid from a github user page
*
* @param userName
*/
export const getGithubUserContribution = async (userName: string) => {
const res = await fetch(`https://github.com/${userName}`);
const resText = await res.text();
// returns the position of the svg elements, accounting for it's transform and it's parent transform
// ( only accounts for translate transform )
const getSvgPosition = (e: cheerio.Element): { x: number; y: number } => {
if (!e || e.tagName === "svg") return { x: 0, y: 0 };
return parseUserPage(resText);
const p = getSvgPosition(e.parent);
if (e.attribs.x) p.x += +e.attribs.x;
if (e.attribs.y) p.y += +e.attribs.y;
if (e.attribs.transform) {
const m = e.attribs.transform.match(
/translate\( *([\.\d]+) *, *([\.\d]+) *\)/
);
if (m) {
p.x += +m[1];
p.y += +m[2];
}
}
return p;
};
type ThenArg<T> = T extends PromiseLike<infer U> ? U : T;

5
packages/github-user-contribution/package.json
View File

@@ -2,10 +2,11 @@
"name": "@snk/github-user-contribution",
"version": "1.0.0",
"dependencies": {
"node-fetch": "2.6.1",
"fast-xml-parser": "3.17.4"
"cheerio": "~1.0.0-rc.3",
"node-fetch": "2.6.1"
},
"devDependencies": {
"@types/cheerio": "0.22.22",
"@types/node-fetch": "2.5.7"
}
}

65
yarn.lock
View File

@@ -576,6 +576,13 @@
"@types/connect" "*"
"@types/node" "*"
"@types/cheerio@0.22.22":
version "0.22.22"
resolved "https://registry.yarnpkg.com/@types/cheerio/-/cheerio-0.22.22.tgz#ae71cf4ca59b8bbaf34c99af7a5d6c8894988f5f"
integrity sha512-05DYX4zU96IBfZFY+t3Mh88nlwSMtmmzSYaQkKN48T495VV1dkHSah6qYyDTN5ngaS0i0VonH37m+RuzSM0YiA==
dependencies:
"@types/node" "*"
"@types/color-name@^1.1.1":
version "1.1.1"
resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0"
@@ -1645,6 +1652,18 @@ char-regex@^1.0.2:
resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf"
integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==
cheerio@~1.0.0-rc.3:
version "1.0.0-rc.3"
resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.3.tgz#094636d425b2e9c0f4eb91a46c05630c9a1a8bf6"
integrity sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==
dependencies:
css-select "~1.2.0"
dom-serializer "~0.1.1"
entities "~1.1.1"
htmlparser2 "^3.9.1"
lodash "^4.15.0"
parse5 "^3.0.1"
chokidar@^2.1.8:
version "2.1.8"
resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917"
@@ -1904,7 +1923,7 @@ cross-spawn@^7.0.0:
shebang-command "^2.0.0"
which "^2.0.1"
css-select@^1.1.0:
css-select@^1.1.0, css-select@~1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/css-select/-/css-select-1.2.0.tgz#2b3a110539c5355f1cd8d314623e870b121ec858"
integrity sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=
@@ -1915,11 +1934,11 @@ css-select@^1.1.0:
nth-check "~1.0.1"
css-tree@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.0.0.tgz#21993fa270d742642a90409a2c0cb3ac0298adf6"
integrity sha512-CdVYz/Yuqw0VdKhXPBIgi8DO3NicJVYZNWeX9XcIuSp9ZoFT5IcleVRW07O5rMjdcx1mb+MEJPknTTEW7DdsYw==
version "1.1.2"
resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.1.2.tgz#9ae393b5dafd7dae8a622475caec78d3d8fbd7b5"
integrity sha512-wCoWush5Aeo48GLhfHPbmvZs59Z+M7k5+B1xDnXbdWNcEF423DoFdqSWE0PM5aNk5nI5cp1q7ms36zGApY/sKQ==
dependencies:
mdn-data "2.0.12"
mdn-data "2.0.14"
source-map "^0.6.1"
css-what@2.1:
@@ -2182,7 +2201,15 @@ dom-serializer@0:
domelementtype "^2.0.1"
entities "^2.0.0"
domelementtype@1, domelementtype@^1.3.1:
dom-serializer@~0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0"
integrity sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==
dependencies:
domelementtype "^1.3.0"
entities "^1.1.1"
domelementtype@1, domelementtype@^1.3.0, domelementtype@^1.3.1:
version "1.3.1"
resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f"
integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==
@@ -2304,7 +2331,7 @@ enquirer@^2.3.4:
dependencies:
ansi-colors "^4.1.1"
entities@^1.1.1:
entities@^1.1.1, entities@~1.1.1:
version "1.1.2"
resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56"
integrity sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==
@@ -2639,11 +2666,6 @@ fast-levenshtein@~2.0.6:
resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917"
integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=
fast-xml-parser@3.17.4:
version "3.17.4"
resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.17.4.tgz#d668495fb3e4bbcf7970f3c24ac0019d82e76477"
integrity sha512-qudnQuyYBgnvzf5Lj/yxMcf4L9NcVWihXJg7CiU1L+oUCq8MUnFEfH2/nXR/W5uq+yvUN1h7z6s7vs2v1WkL1A==
faye-websocket@^0.10.0:
version "0.10.0"
resolved "https://registry.yarnpkg.com/faye-websocket/-/faye-websocket-0.10.0.tgz#4e492f8d04dfb6f89003507f6edbf2d501e7c6f4"
@@ -3049,7 +3071,7 @@ html-webpack-plugin@5.0.0-alpha.10:
pretty-error "^2.1.1"
tapable "2.0.0"
htmlparser2@^3.3.0:
htmlparser2@^3.3.0, htmlparser2@^3.9.1:
version "3.10.1"
resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f"
integrity sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==
@@ -4161,7 +4183,7 @@ lodash.sortby@^4.7.0:
resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=
lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.20:
lodash@^4.15.0, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.20:
version "4.17.20"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52"
integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==
@@ -4214,10 +4236,10 @@ map-visit@^1.0.0:
dependencies:
object-visit "^1.0.0"
mdn-data@2.0.12:
version "2.0.12"
resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.12.tgz#bbb658d08b38f574bbb88f7b83703defdcc46844"
integrity sha512-ULbAlgzVb8IqZ0Hsxm6hHSlQl3Jckst2YEQS7fODu9ilNWy2LvcoSY7TRFIktABP2mdppBioc66va90T+NUs8Q==
mdn-data@2.0.14:
version "2.0.14"
resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.14.tgz#7113fc4281917d63ce29b43446f701e68c25ba50"
integrity sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==
media-typer@0.3.0:
version "0.3.0"
@@ -4827,6 +4849,13 @@ parse5@5.1.1:
resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178"
integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==
parse5@^3.0.1:
version "3.0.3"
resolved "https://registry.yarnpkg.com/parse5/-/parse5-3.0.3.tgz#042f792ffdd36851551cf4e9e066b3874ab45b5c"
integrity sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==
dependencies:
"@types/node" "*"
parseurl@~1.3.2, parseurl@~1.3.3:
version "1.3.3"
resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4"