feat: allow citgm job comparison (#455)

Author: codebytere

bin/ncu-ci

@@ -21,6 +21,7 @@ const {
   CommitBuild,
   CITGMBuild,
   DailyBuild,
+  CITGMComparisonBuild,
   HealthBuild,
   listBuilds,
   FailureAggregator,
@@ -113,7 +114,7 @@ const argv = yargs
     builder: (yargs) => {
       yargs
         .positional('jobid', {
-          describe: 'id of the job',
+          describe: 'id of the first job',
           type: 'number'
         });
     },
@@ -132,13 +133,17 @@ const argv = yargs
     handler
   })
   .command({
-    command: 'citgm <jobid>',
+    command: 'citgm <jobid> [jobid2]',
     desc: 'Show results of a citgm-smoker CI job',
     builder: (yargs) => {
       yargs
         .positional('jobid', {
           describe: 'id of the job',
           type: 'number'
+        })
+        .positional('jobid2', {
+          describe: 'id of the second job, if doing a comparison',
+          type: 'number'
         });
     },
     handler
@@ -166,12 +171,14 @@ const argv = yargs
   })
   .demandCommand(1, 'must provide a valid command')
   .option('copy', {
+    describe: 'Write the results as markdown to clipboard',
     default: false,
-    describe: 'Write the results as markdown to clipboard'
+    type: 'boolean'
   })
   .option('nobuild', {
-    describe: 'If running cigtm, whether or not the CITGM job is citgm-nobuild',
-    type: 'boolean'
+    describe: 'If running cigtm, whether or not jobid is citgm-nobuild.',
+    type: 'boolean',
+    default: false
   })
   .option('json <path>', {
     type: 'string',
@@ -241,7 +248,11 @@ class CICommand {
           break;
         case CITGM:
         case CITGM_NOBUILD:
-          build = new CITGMBuild(cli, request, job);
+          if (job.jobid2) {
+            build = new CITGMComparisonBuild(cli, request, job);
+          } else {
+            build = new CITGMBuild(cli, request, job);
+          }
           break;
         case BENCHMARK:
           build = new BenchmarkRun(cli, request, job.jobid);
@@ -361,10 +372,13 @@ class JobCommand extends CICommand {
   }
 
   async initialize() {
-    this.queue.push({
+    const { queue, argv } = this;
+
+    queue.push({
       type: commandToType[this.command],
-      jobid: this.argv.jobid,
-      noBuild: this.argv.nobuild || false
+      jobid: argv.jobid,
+      jobid2: argv.jobid2,
+      noBuild: this.argv.nobuild
     });
   }
 }

docs/ncu-ci.md

@@ -26,9 +26,10 @@ Commands:
 
 Options:
   --version          Show version number                               [boolean]
-  --copy             Write the results as markdown to clipboard [default: false]
-  --nobuild          If running cigtm, whether or not the CITGM job is
-                     citgm-nobuild                                     [boolean]
+  --copy             Write the results as markdown to clipboard
+                                                      [boolean] [default: false]
+  --nobuild          If running cigtm, whether or not jobid is citgm-nobuild.
+                                                      [boolean] [default: false]
   --json <path>      Write the results as json to <path>                [string]
   --markdown <path>  Write the results as markdown to <path>            [string]
   --help             Show help                                         [boolean]
@@ -230,13 +231,13 @@ Notifying upstream projects of job completion
 Finished: SUCCESS
 ```
 
-### `ncu-ci citgm <jobid>`
+### `ncu-ci citgm <jobid> [jobid2]`
 
-`ncu-ci citgm <jobid>` shows the results of a given citgm-smoker job. See `ncu-ci citgm --help` for more.
+`ncu-ci citgm <jobid> [jobid2]` shows the results of a given citgm-smoker job, with the option to compare per-platform results of two jobs. You See `ncu-ci citgm --help` for more.
 
 Example:
 ```
-node on git:master ❯ ncu-ci citgm 2400                                         10:25AM
+node on git:master ❯ ncu-ci citgm 2400
 --------------------------------------------------------------------------------
 [1/1] Running CITGM: 2400
 --------------------------------------------------------------------------------
@@ -264,12 +265,58 @@ Author     Shelley Vohr <shelley.vohr@gmail.com>
 └────────────────────────┴───────────────────────┴───────────────────────┴─────────────────────────┴─────────────────────┴─────────────────┴────────────────────┘
 ```
 
+Comparison Example:
+```sh
+node-core-utils on git:allow-citgm-comparison ❯ ncu-ci citgm 2392 2390
+--------------------------------------------------------------------------------
+[1/1] Running CITGM: 2392
+--------------------------------------------------------------------------------
+✔  Summary data downloaded
+✔  Results data downloaded
+✔  Summary data downloaded
+✔  Results data downloaded
+----------------------------------- Summary ------------------------------------
+Result     FAILURE
+URL        https://ci.nodejs.org/job/citgm-smoker/2392/
+Source     https://api.github.com/repos/nodejs/node/git/refs/heads/v12.x
+Commit     [feed95cd4c2c] Working on v12.18.1
+Date       2020-06-02 20:27:47 +0200
+Author     Michaël Zasso <targos@protonmail.com>
+----------------------------------- Summary ------------------------------------
+Result     FAILURE
+URL        https://ci.nodejs.org/job/citgm-smoker/2390/
+Source     https://github.com/nodejs/node/pull/33811/
+Commit     [9a60117875dd] 2020-06-16, Version 12.18.1 'Erbium' (LTS)
+Date       2020-06-09 20:23:09 -0700
+Author     Shelley Vohr <shelley.vohr@gmail.com>
+----------------------------------- Results ------------------------------------
+
+
+
+FAILURE: 5 failures in 2390 not present in 2392
+
+
+┌────────────────────────┬───────────────────────────┬────────────────────────────┐
+│        (index)         │             0             │             1              │
+├────────────────────────┼───────────────────────────┼────────────────────────────┤
+│     centos7-ppcle      │      'multer-v1.4.2'      │                            │
+│   fedora-latest-x64    │    'spawn-wrap-v2.0.0'    │                            │
+│ fedora-last-latest-x64 │                           │                            │
+│       debian9-64       │ 'express-session-v1.17.1' │ 'yeoman-generator-v4.10.1' │
+│        osx1014         │                           │                            │
+│      rhel7-s390x       │  'torrent-stream-v1.2.0'  │                            │
+│      aix71-ppc64       │                           │                            │
+│     ubuntu1604-64      │                           │                            │
+│     ubuntu1804-64      │                           │                            │
+└────────────────────────┴───────────────────────────┴────────────────────────────┘
+```
+
 ### `ncu-ci daily`
 
 `ncu-ci daily` show recent results of `node-daily-master`. You can also aggregate the results by passing `--cache`, or limit the maximum number of CIs jobs to get data from with `--limit=N`. See `ncu-ci daily --help` for more.
 
 ```sh
-node on git:master ❯ ncu-ci daily                                            12:14PM
+node on git:master ❯ ncu-ci daily
 ✔  Done--------------------------------------------------------------------------------
 [1/16] Running health
 --------------------------------------------------------------------------------

lib/ci/build-types/citgm_build.js

@@ -0,0 +1,182 @@
+'use strict';
+
+const { TestBuild } = require('./test_build');
+const { flatten } = require('../../utils');
+const { getNodeName, pad } = require('../ci_utils');
+
+const {
+  CITGM_MAIN_TREE,
+  CITGM_REPORT_TREE
+} = require('../jenkins_constants');
+const {
+  FAILURE_TYPES: { NCU_FAILURE },
+  FAILURE_CONSTRUCTORS: {
+    [NCU_FAILURE]: NCUFailure
+  }
+} = require('../ci_failure_parser');
+
+class CITGMBuild extends TestBuild {
+  constructor(cli, request, job) {
+    const { jobid, noBuild } = job;
+    const path = noBuild
+      ? `job/citgm-smoker-nobuild/${jobid}/`
+      : `job/citgm-smoker/${jobid}/`;
+
+    const tree = CITGM_MAIN_TREE;
+
+    super(cli, request, path, tree);
+
+    this.id = jobid;
+    this.noBuild = noBuild;
+  }
+
+  async getResults() {
+    const { apiUrl } = this;
+
+    let headerData;
+    try {
+      headerData = await this.getBuildData('Summary');
+    } catch (err) {
+      this.failures = [
+        new NCUFailure({ url: this.apiUrl }, err.message)
+      ];
+      return this.failures;
+    }
+    const { result } = headerData;
+
+    this.setBuildData(headerData);
+
+    // CITGM jobs store results in a different location than
+    // they do summary data, so we need to update the endpoint
+    // and issue a second API call in order to fetch result data.
+    this.tree = CITGM_REPORT_TREE;
+    this.updatePath(true);
+    let resultData;
+    try {
+      resultData = await this.getBuildData('Results');
+    } catch (err) {
+      this.failures = [
+        new NCUFailure({ url: apiUrl }, err.message)
+      ];
+      return this.failures;
+    }
+
+    this.results = this.parseResults(resultData);
+
+    // Update id again so that it correctly displays in Summary output.
+    this.updatePath(false);
+
+    return { result };
+  }
+
+  parseResults(data) {
+    const { childReports, totalCount, skipCount, failCount } = data;
+    const results = { all: {}, failures: {}, statistics: {} };
+    const failureStatuses = ['FAILED', 'REGRESSION'];
+
+    const passCount = totalCount - failCount - skipCount;
+    results.statistics.passed = passCount;
+    results.statistics.total = totalCount;
+    results.statistics.skipped = skipCount;
+    results.statistics.failed = failCount;
+
+    childReports.forEach(platform => {
+      const cases = flatten(platform.result.suites[0].cases);
+      const url = platform.child.url;
+      const nodeName = getNodeName(url);
+
+      results.all[nodeName] = { url, modules: cases };
+
+      const failedModules = cases.filter(c => {
+        return failureStatuses.includes(c.status);
+      });
+
+      results.failures[nodeName] = { url, modules: failedModules };
+    });
+
+    return results;
+  }
+
+  updatePath(testReport) {
+    const { id, noBuild } = this;
+    if (testReport) {
+      this.path = noBuild
+        ? `job/citgm-smoker-nobuild/${id}/testReport/`
+        : `job/citgm-smoker/${id}/testReport/`;
+    } else {
+      this.path = noBuild
+        ? `job/citgm-smoker-nobuild/${id}/`
+        : `job/citgm-smoker/${id}/`;
+    }
+  }
+
+  displayBuilds() {
+    const { cli, results } = this;
+    const { failed, skipped, passed, total } = results.statistics;
+
+    cli.separator('Statistics');
+    console.table({
+      Failed: failed,
+      Skipped: skipped,
+      Passed: passed,
+      Total: total
+    });
+
+    cli.separator('Failures');
+    const output = {};
+    for (const platform in results.failures) {
+      const modules = results.failures[platform].modules;
+      const failures = modules.map(f => f.name);
+
+      output[platform] = failures;
+    }
+
+    console.table(output);
+  }
+
+  formatAsJson() {
+    const { jobUrl, results, sourceURL } = this;
+
+    const result = {
+      source: sourceURL,
+      upstream: jobUrl,
+      ...results.statistics,
+      ...results.failures
+    };
+
+    return JSON.parse(JSON.stringify(result));
+  }
+
+  formatAsMarkdown() {
+    const { jobUrl, result, results, id } = this;
+
+    let output = `# CITGM Data for [${id}](${jobUrl})\n\n`;
+
+    const { failed, skipped, passed, total } = results.statistics;
+
+    output += `## Statistics for job [${id}](${jobUrl})\n\n`;
+    output += '|  FAILED  |  SKIPPED  |  PASSED  |  TOTAL  |\n';
+    output += '| -------- | --------- | -------- | ------- |\n';
+    output += `| ${pad(failed, 8)} | ${pad(skipped, 9)} |`;
+    output += ` ${pad(passed, 8)} | ${pad(total, 7)} |\n\n`;
+
+    if (result === 'success') {
+      output += `Job [${id}](${jobUrl}) is green.`;
+      return output;
+    }
+
+    output += `## Failures in job [${id}](${jobUrl})\n\n`;
+    for (const failure in results.failures) {
+      const data = results.failures[failure];
+      output += `### [${failure}](${data.url})\n\n`;
+
+      const failures = data.modules.map(f => `* ${f.name}`);
+      const items = failures.length > 0 ? `${failures.join('\n')}` : 'None.';
+      output += `${items}\n\n`;
+    }
+
+    return output;
+  }
+}
+
+module.exports = { CITGMBuild };