Added GitHub issue caching.

admin/cmds/cache-github.js

@@ -0,0 +1,10 @@
+"use strict";
+
+const config = require("../config");
+const { syncIssues } = require("../github");
+
+(async function() {
+    const user = await config.get("github-user");
+    const password = await config.get("github-readonly");
+    await syncIssues(user, password);
+})();

admin/cmds/grep-github.js

@@ -0,0 +1,142 @@
+"use strict";
+
+const { colorify } = require("../log");
+const { getIssues } = require("../github");
+const { repeat } = require("../utils");
+
+const Options = {
+    "body": 1,
+    "end": 1,
+    "issue": 1,
+    "start": 1,
+    "title": 1,
+    "user": 1,
+};
+
+const Flags = {
+    "open": 1,
+    "match-case": 1,
+};
+
+(async function() {
+    const options = { };
+    for (let i = 2; i < process.argv.length; i++) {
+        const option = process.argv[i];
+        if (option.substring(0, 2) === "--") {
+            const comps = option.substring(2).split(/=/);
+            if (Flags[comps[0]]) {
+                if (comps[1] != null) { throw new Error("Invalid flag: " + option); }
+                options[comps[0]] = true;
+            } else if (Options[comps[0]]) {
+                if (comps[1] == null) {
+                    options[comps[0]] = process.argv[++i];
+                    if (options[comps[0]] == null) {
+                        throw new Error("Missing option value: " + option);
+                    }
+                } else {
+                    options[comps[0]] = comps[1];
+                }
+            } else {
+                throw new Error("Unexpected option: " + option);
+            }
+        } else {
+            throw new Error("Unexpected argument: " + option);
+        }
+    }
+
+    if (options["title"]) { options.title = new RegExp(options.title, (options["match-case"] ? "": "i")); }
+    if (options["body"]) { options.body = new RegExp(options.title, (options["match-case"] ? "": "i")); }
+    if (options["start"]) {
+        if (options["start"].match(/^[0-9]{4}-[0-9]{2}-[0-9{2}]$/)) {
+            throw new Error("Expected YYYY-MM-DD");
+        }
+    }
+    if (options["end"]) {
+        if (options["end"].match(/^[0-9]{4}-[0-9]{2}-[0-9{2}]$/)) {
+            throw new Error("Expected YYYY-MM-DD");
+        }
+    }
+
+    const count = { issues: 0, comments: 0, code: 0, responses: 0 };
+
+    const issues = await getIssues();
+    issues.forEach((issue) => {
+        const info = issue.issue;
+        const comments = issue.comments;
+
+        if (options.issue && parseInt(options.issue) != info.number) { return; }
+        if (options.open && info.state !== "open") { return; }
+        if (options.title && !info.title.match(options.title)) { return; }
+        if (options.body) {
+            const body = info.body + "\n" + comments.map((c) => (c.body)).join("\n");
+            if (!body.match(options.body)) {
+                return;
+            }
+        }
+        if (options.user) {
+            const users = comments.map((c) => (c.user.login));
+            users.push(info.user.login);
+            if (users.indexOf(options.user) === -1) {
+                return;
+            }
+        }
+
+        const dates = comments.map((c) => (c.created_at.split("T")[0]));
+        dates.push(info.created_at.split("T")[0]);
+
+        if (options.start) {
+            if (dates.filter((d) => (d >= options.start)).length === 0) { return; }
+        }
+        if (options.end) {
+            if (dates.filter((d) => (d <= options.start)).length === 0) { return; }
+        }
+
+        count.issues++;
+
+        console.log(colorify(repeat("=", 70), "bold"))
+        console.log(colorify("Issue:", "bold"), info.title, ` (#${ info.number })`);
+        console.log(colorify("User:","bold"), colorify(info.user.login, "blue"));
+        console.log(colorify("State:", "bold"), info.state);
+        if (info.created_at === info.updated_at) {
+            console.log(colorify("Created:", "bold"), info.created_at);
+        } else {
+            console.log(colorify("Created:", "bold"), info.created_at, ` (updated: ${ info.updated_at })`);
+        }
+        info.body.trim().split("\n").forEach((line) => {
+            console.log("  " + line);
+        });
+
+        if (comments.length) {
+            comments.forEach((info) => {
+                if (options.start && info.created_at < options.start) { return ; }
+                if (options.end && info.created_at > options.end) { return; }
+                count.comments++;
+                if (options.user && info.user.login !== options.user) { return; }
+                count.responses++;
+                if (info.body.indexOf("`") >= 0) { count.code++; }
+                console.log(colorify(repeat("-", 70), "bold"));
+                console.log(colorify("User:", "bold"), colorify(info.user.login, "green"));
+                if (info.created_at === info.updated_at) {
+                    console.log(colorify("Created:", "bold"), info.created_at);
+                } else {
+                    console.log(colorify("Created:", "bold"), info.created_at, ` (updated: ${ info.updated_at })`);
+                }
+                info.body.trim().split("\n").forEach((line) => {
+                    console.log("  " + line);
+                });
+            });
+        }
+    });
+
+    console.log(colorify(repeat("=", 70), "bold"))
+
+    // @TODO: Add stats on new/closed issues
+    //if (options.user) {
+    //    console.log(`${ count.responses } responses (${ count.code } w/ code) on ${ count.comments } comments across ${ count.issues } issues.`);
+    //} else {
+        console.log(`${ count.comments } comment${ (count.comments !== 1) ? "s": "" } across ${ count.issues } issue${ (count.issues !== 1) ? "s": ""}.`);
+    //}
+
+})().catch((error) => {
+    console.log("Error: " + error.message);
+});

admin/github.js

@@ -0,0 +1,134 @@
+"use strict";
+
+const fs = require("fs");
+const { resolve } = require("path");
+const zlib = require("zlib");
+
+const { id } = require("../packages/hash");
+const { fetchJson } = require("../packages/web");
+
+const CacheDir = resolve(__dirname, "../github-cache/");
+
+function addResponse(result, response) {
+    return { result, response };
+}
+
+function loadFile(filename) {
+    return JSON.parse(zlib.gunzipSync(fs.readFileSync(filename)).toString());
+    //return JSON.parse(fs.readFileSync(filename).toString());
+}
+
+// @TODO: atomic
+function saveFile(filename, content) {
+    fs.writeFileSync(filename, zlib.gzipSync(JSON.stringify(content)));
+    //fs.writeFileSync(filename, JSON.stringify(content));
+}
+
+function mockFetchJson(url, body, headers) {
+    return {
+        result: null,
+        response: {
+            statusCode: 304
+        }
+    }
+}
+
+async function _fetchGitHub(user, password, fetchJson, url) {
+    const result = [ ];
+    while (true) {
+
+        const filename = resolve(CacheDir, id(url).substring(2, 14));
+
+        const headers = {
+            "User-Agent": "ethers-io",
+        };
+
+        let items = null;
+        let link = null;
+        try {
+            const data = loadFile(filename);
+            headers["if-none-match"] = data.etag;
+            items = data.items;
+            link = data.link;
+        } catch (error) {
+            if (error.code !== "ENOENT") { throw error; }
+        }
+
+        const fetch = await fetchJson({
+            url: url,
+            user: user,
+            password: password,
+            headers: headers
+        }, null, addResponse);
+
+        // Cached response is good; use it!
+        if (fetch.response.statusCode !== 304) {
+            items = fetch.result;
+            if (fetch.response.headers) {
+                link = (fetch.response.headers.link || null);
+            }
+            if (fetch.response.headers.etag){
+                saveFile(filename, {
+                    timestamp: (new Date()).getTime(),
+                    url: url,
+                    link: link,
+                    etag: fetch.response.headers.etag,
+                    items: items,
+                    version: 1
+                });
+            }
+        }
+
+        items.forEach((item) => { result.push(item)});
+
+        url = null;
+        (link || "").split(",").forEach((item) => {
+            if (item.indexOf('rel="next"') >= 0) {
+                const match = item.match(/<([^>]*)>/);
+                if (match) { url = match[1]; }
+            }
+        });
+
+        if (!url) { break; }
+    }
+    return result;
+}
+
+async function fetchGitHub(user, password, url, cacheOnly) {
+    if (cacheOnly) {
+        return await _fetchGitHub("none", "none", mockFetchJson, url);
+    }
+
+    const results = await _fetchGitHub(user, password, fetchJson, url);
+    return results;
+}
+
+
+async function _getIssues(user, password) {
+    const cacheOnly = (user == null);
+
+    let issues = await fetchGitHub(user, password, "https://api.github.com/repos/ethers-io/ethers.js/issues?state=all&per_page=100", cacheOnly)
+    if (!cacheOnly) { console.log(`Found ${ issues.length } issues`); }
+    const result = [ ];
+    for (let i = 0; i < issues.length; i++) {
+        const issue = issues[i];
+        let comments = await fetchGitHub(user, password, issue.comments_url, cacheOnly);
+        result.push({ issue, comments});
+        if (!cacheOnly) { console.log(`  Issue ${ issue.number }: ${ comments.length } comments`); }
+    }
+    result.sort((a, b) => (a.issue.number - b.issue.number));
+    return result;
+}
+
+function getIssues() {
+    return _getIssues();
+}
+
+function syncIssues(user, password) {
+    return _getIssues(user, password);
+}
+
+module.exports = {
+    getIssues,
+    syncIssues,
+}