Checkpoint: functions for loading the config and rule functions from …

….fensak repository

Signed-off-by: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com>

.github/workflows/lint-test.yml

@@ -33,6 +33,7 @@ jobs:
         run: deno test --allow-net --allow-env --allow-read --reporter=junit --junit-path=./report.xml
         env:
           FENSAK_GITHUB_WEBHOOK_SECRET: ${{ secrets.FENSAK_GITHUB_WEBHOOK_SECRET }}
+          FENSAK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: report
         uses: mikepenz/action-junit-report@75b84e78b3f0aaea7ed7cf8d1d100d7f97f963ec # v4.0.0

config/custom-environment-variables.json5

@@ -1,6 +1,7 @@
 {
   github: {
     webhookSecret: "FENSAK_GITHUB_WEBHOOK_SECRET",
+    apiToken: "FENSAK_GITHUB_TOKEN",
   },
   faunadb: {
     name: "FENSAK_FAUNADB_NAME",

config/default.json5

@@ -8,6 +8,9 @@
   github: {
     // The secret string used to authenticate GitHub webhook requests coming in.
     webhookSecret: "",
+    // An optional API token to use to authenticate GitHub API requests. Only used in dev and test mode (in production,
+    // the GitHub app configurations are used).
+    apiToken: "",
   },
 
   /**

deps.ts

@@ -3,6 +3,7 @@ export {
   toHashString,
 } from "https://deno.land/std@0.202.0/crypto/mod.ts";
 export * as hex from "https://deno.land/std@0.202.0/encoding/hex.ts";
+export * as base64 from "https://deno.land/std@0.202.0/encoding/base64.ts";
 export * as path from "https://deno.land/std@0.202.0/path/mod.ts";
 export * as toml from "https://deno.land/std@0.202.0/toml/mod.ts";
 export * as yaml from "https://deno.land/std@0.202.0/yaml/mod.ts";

fskconfig/loader_github.ts

@@ -0,0 +1,178 @@
+import { base64, Octokit, path } from "../deps.ts";
+
+import { compileRuleFn, RuleFnSourceLang } from "../udr/mod.ts";
+
+import type { ComputedFensakConfig, OrgConfig, RuleLookup } from "./types.ts";
+import { getRuleLang, parseConfigFile } from "./parser.ts";
+
+const fensakCfgRepoName = ".fensak";
+
+interface IGitFileInfo {
+  filename: string;
+  gitSHA: string;
+}
+
+/**
+ * Loads a Fensak configuration from GitHub. This looks up the configuration from the repository `.fensak` in the
+ * organization.
+ *
+ * @param clt An authenticated Octokit instance.
+ * @param owner The GitHub owner to load the config for.
+ */
+export async function loadConfigFromGitHub(
+  clt: Octokit,
+  owner: string,
+): Promise<ComputedFensakConfig> {
+  const { data: repo } = await clt.repos.get({
+    owner: owner,
+    repo: fensakCfgRepoName,
+  });
+  const defaultBranch = repo.default_branch;
+  const { data: ref } = await clt.git.getRef({
+    owner: owner,
+    repo: fensakCfgRepoName,
+    ref: `heads/${defaultBranch}`,
+  });
+  const headSHA = ref.object.sha;
+
+  // TODO
+  // Load from cache if the SHA is the same.
+
+  const fileSHALookup = await getFileSHALookup(clt, owner, headSHA);
+  const cfgFinfo = getConfigFinfo(fileSHALookup);
+  if (!cfgFinfo) {
+    throw new Error(
+      `could not find fensak config file in the '${owner}/.fensak' repo`,
+    );
+  }
+
+  const orgCfgContents = await loadFileContents(clt, owner, cfgFinfo);
+  const orgCfg = parseConfigFile(cfgFinfo.filename, orgCfgContents);
+  const ruleLookup = await loadRuleFiles(clt, owner, orgCfg, fileSHALookup);
+
+  return {
+    orgConfig: orgCfg,
+    ruleLookup: ruleLookup,
+    gitSHA: headSHA,
+  };
+}
+
+/**
+ * Create a lookup table that maps file paths in a repository tree to the file sha.
+ */
+async function getFileSHALookup(
+  clt: Octokit,
+  owner: string,
+  sha: string,
+): Promise<Record<string, string>> {
+  const { data: tree } = await clt.git.getTree({
+    owner: owner,
+    repo: fensakCfgRepoName,
+    tree_sha: sha,
+    recursive: "true",
+  });
+  const out: Record<string, string> = {};
+  for (const f of tree.tree) {
+    if (!f.path || !f.sha) {
+      continue;
+    }
+    out[f.path] = f.sha;
+  }
+  return out;
+}
+
+/**
+ * Get config file name and sha in the repo by walking the repository tree.
+ */
+function getConfigFinfo(
+  repoTreeLookup: Record<string, string>,
+): IGitFileInfo | null {
+  for (const fpath in repoTreeLookup) {
+    const fpathBase = path.basename(fpath);
+    const fpathExt = path.extname(fpathBase);
+    if (fpathBase === `fensak${fpathExt}`) {
+      const fsha = repoTreeLookup[fpath];
+      return {
+        filename: fpath,
+        gitSHA: fsha,
+      };
+    }
+  }
+  return null;
+}
+
+/**
+ * Load the contents of the given file in the `.fensak` repository.
+ */
+async function loadFileContents(
+  clt: Octokit,
+  owner: string,
+  finfo: IGitFileInfo,
+): Promise<string> {
+  const { data: file } = await clt.git.getBlob({
+    owner: owner,
+    repo: fensakCfgRepoName,
+    file_sha: finfo.gitSHA,
+  });
+
+  if (file.encoding !== "base64") {
+    throw new Error(
+      `unknown encoding from github blob when retrieving ${finfo.filename}: ${file.encoding}`,
+    );
+  }
+  const contentsBytes = base64.decode(file.content);
+  return new TextDecoder().decode(contentsBytes);
+}
+
+/**
+ * Load the referenced rule files in the org config from the `.fensak` repository so that they can be cached.
+ */
+async function loadRuleFiles(
+  clt: Octokit,
+  owner: string,
+  orgCfg: OrgConfig,
+  fileSHALookup: Record<string, string>,
+): Promise<RuleLookup> {
+  const ruleFilesToLoad: Record<string, RuleFnSourceLang> = {};
+  for (const repoName in orgCfg.repos) {
+    const repoCfg = orgCfg.repos[repoName];
+    if (ruleFilesToLoad[repoCfg.ruleFile]) {
+      // Skip because it is already accounted for
+      continue;
+    }
+
+    // This is redundant and unnecessary, but it makes the compiler happy.
+    if (!repoCfg.ruleLang) {
+      repoCfg.ruleLang = getRuleLang(repoCfg.ruleFile);
+    }
+
+    ruleFilesToLoad[repoCfg.ruleFile] = repoCfg.ruleLang;
+  }
+
+  const out: RuleLookup = {};
+  for (const fname in ruleFilesToLoad) {
+    const sha = fileSHALookup[fname];
+    if (!sha) {
+      throw new Error(
+        `could not find referenced rule file '${fname}' in '.fensak' repository`,
+      );
+    }
+
+    const ruleLang = ruleFilesToLoad[fname];
+
+    // NOTE
+    // Ideally we can asynchronously fetch the contents here, but GitHub API is very strict about parallel calls and
+    // rate limiting, so we have to resort to a lousy loop here.
+    const contents = await loadFileContents(clt, owner, {
+      filename: fname,
+      gitSHA: sha,
+    });
+    const compiledContents = compileRuleFn(contents, ruleLang);
+
+    out[fname] = {
+      sourceGitHash: sha,
+      compiledRule: compiledContents,
+    };
+  }
+  return out;
+}

fskconfig/loader_github_test.ts

@@ -0,0 +1,51 @@
+import { assert, assertEquals } from "../test_deps.ts";
+import { config, Octokit } from "../deps.ts";
+
+import { RuleFnSourceLang } from "../udr/mod.ts";
+
+import { loadConfigFromGitHub } from "./loader_github.ts";
+
+const token = config.get("github.apiToken");
+let octokit: Octokit;
+if (token) {
+  octokit = new Octokit({ auth: token });
+} else {
+  octokit = new Octokit();
+}
+
+Deno.test("loadConfigFromGitHub for fensak-test example repo", async () => {
+  const cfg = await loadConfigFromGitHub(octokit, "fensak-test");
+  assertEquals(cfg.gitSHA, "4c35fe73411fd4a57cd45b0621d63638536425fc");
+  assertEquals(cfg.orgConfig, {
+    repos: {
+      "test-github-webhooks": {
+        ruleFile: "subfolder/allow_readme_changes.js",
+        ruleLang: RuleFnSourceLang.ES6,
+      },
+      "test-fensak-rules-engine": {
+        ruleFile: "app_deploy_rule.ts",
+        ruleLang: RuleFnSourceLang.Typescript,
+      },
+    },
+  });
+
+  // Test that the ruleLookup record contains rules for the expected rule functions.
+  //
+  // Ideally we can use the assertion functions to test this, but for some bizarre reason, even though the object
+  // materializes, the assertion functions see that the object is undefined.
+  const appDeployRule = cfg.ruleLookup["app_deploy_rule.ts"];
+  if (!appDeployRule) {
+    assert(false, "The rule app_deploy_rule.ts was not successfully compiled");
+  }
+  const allowReadmeChanges =
+    cfg.ruleLookup["subfolder/allow_readme_changes.js"];
+  if (!allowReadmeChanges) {
+    assert(
+      false,
+      "The rule subfolder/allow_readme_changes.js was not successfully compiled",
+    );
+  }
+
+  // TODO
+  // add some basic testing for the compiled rule source
+});

fskconfig/parser.ts

@@ -1,8 +1,9 @@
 import { Ajv, path, toml, yaml } from "../deps.ts";
 
-import type { OrgConfig } from "./types.ts";
 import { RuleFnSourceLang } from "../udr/mod.ts";
 
+import type { OrgConfig } from "./types.ts";
+
 const __dirname = path.dirname(path.fromFileUrl(import.meta.url));
 const cfgSchemaTxt = await Deno.readTextFile(
   path.join(__dirname, "./schema.json"),
@@ -63,7 +64,7 @@ export function parseConfigFile(
   return typedData;
 }
 
-function getRuleLang(ruleFname: string): RuleFnSourceLang {
+export function getRuleLang(ruleFname: string): RuleFnSourceLang {
   const ext = path.extname(ruleFname);
   switch (ext) {
     default:

fskconfig/types.ts

@@ -2,21 +2,55 @@ import { RuleFnSourceLang } from "../udr/mod.ts";
 
 /**
  * The configuration for an organization.
+ * @property repos The mapping of repo names (scoped to the org) to the corresponding repository configuration.
  */
 interface OrgConfig {
-  // The mapping of repo names (scoped to the org) to the corresponding repository configuration.
   repos: Record<string, RepoConfig>;
 }
 
 /**
  * The configuration for a specific repository.
+ * @property ruleFile The path (relative to the repo root) to the file to use for the rules source.
+ * @property ruleLang The language that the rules source is written in. If omitted, the language is derived from the
+ *                    source file extension. Note that we will always assume ES6 for js files.
  */
 interface RepoConfig {
-  // The path (relative to the repo root) to the file to use for the rules source.
   ruleFile: string;
-  // The language that the rules source is written in. If omitted, the language is derived from the source file
-  // extension. Note that we will always assume ES6 for js files.
   ruleLang?: RuleFnSourceLang;
 }
 
-export type { OrgConfig, RepoConfig };
+/**
+ * The computed Fensak config for a particular organization.
+ * @property orgConfig The user provided configuration defining which rules apply to which repo.
+ * @property ruleLookup A lookup table that maps the rules in the org repo to their compiled definition.
+ * @property gitSHA The commit sha used for retrieving the configuration. Used for cache busting.
+ */
+interface ComputedFensakConfig {
+  orgConfig: OrgConfig;
+  ruleLookup: RuleLookup;
+  gitSHA: string;
+}
+
+/**
+ * The compiled ES5 compatible rule source.
+ * @property sourceGitHash The git hash of the original source file. Used for cache busting.
+ * @property compiledRule The compiled, ES5 compatible rule source file.
+ */
+interface CompiledRuleSource {
+  sourceGitHash: string;
+  compiledRule: string;
+}
+
+/**
+ * A lookup table mapping source file names to the corresponding compiled file contents. This is used to quickly
+ * retrieve the source contents for a particular Org.
+ */
+type RuleLookup = Record<string, CompiledRuleSource>;
+
+export type {
+  CompiledRuleSource,
+  ComputedFensakConfig,
+  OrgConfig,
+  RepoConfig,
+  RuleLookup,
+};

patch/from_github.ts

@@ -1,4 +1,5 @@
 import { crypto, hex, Octokit, toHashString } from "../deps.ts";
+
 import { parseUnifiedDiff } from "./patch.ts";
 import { IPatch, PatchOp } from "./patch_types.ts";
 import { SourcePlatform } from "./from.ts";

patch/from_github_test.ts

@@ -1,11 +1,17 @@
 import { assertEquals, assertNotEquals } from "../test_deps.ts";
-import { Octokit } from "../deps.ts";
+import { config, Octokit } from "../deps.ts";
 
 import { IPatch, LineOp, PatchOp } from "./patch_types.ts";
 import { patchFromGitHubPullRequest } from "./from_github.ts";
 import type { IGitHubRepository } from "./from_github.ts";
 
-const octokit = new Octokit();
+const token = config.get("github.apiToken");
+let octokit: Octokit;
+if (token) {
+  octokit = new Octokit({ auth: token });
+} else {
+  octokit = new Octokit();
+}
 const testRepo: IGitHubRepository = {
   owner: "fensak-test",
   name: "test-fensak-rules-engine",

test_deps.ts

@@ -1,6 +1,7 @@
 export {
   assert,
   assertEquals,
+  assertExists,
   assertFalse,
   assertNotEquals,
   assertRejects,