tests: run smoke tests in parallel jobs

GoogleChrome · Jun 18, 2020 · 6c8bf7e · 6c8bf7e
1 parent cfce99a
commit 6c8bf7e
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 28 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -8,11 +8,7 @@ on: [pull_request]
 
 jobs:
   ci:
-
     runs-on: ubuntu-latest
-    strategy:
-      # e.g. if lint fails, continue to the unit tests anyway
-      fail-fast: false
 
     steps:
     - name: git clone
@@ -23,7 +19,7 @@ jobs:
       with:
         node-version: 10.x
 
-    - name: Setup protoc
+    - name: Set up protoc
       uses: arduino/setup-protoc@7ad700d
       with:
         version: '3.7.1'
@@ -38,19 +34,6 @@ jobs:
         python -m pip install --upgrade pip
         pip install protobuf==3.7.1
 
-    # Cache yarn deps. thx https://github.com/actions/cache/blob/master/examples.md#node---yarn
-    - name: Get yarn cache directory path
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - name: Set up node_modules cache
-      uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-
     - run: yarn --frozen-lockfile
     - run: yarn build-all
     - run: yarn diff:sample-json
@@ -60,17 +43,26 @@ jobs:
 
     # Run tests that require headfull Chrome.
     - run: sudo apt-get install xvfb
-    - run: xvfb-run --auto-servernum yarn unit
-    - run: xvfb-run --auto-servernum yarn test-clients
-    - run: xvfb-run --auto-servernum yarn smoke --debug -j=1 --retries=2
-    - run: xvfb-run --auto-servernum yarn test-bundle
-    - run: xvfb-run --auto-servernum yarn test-docs
+    - name: yarn unit
+      run: xvfb-run --auto-servernum yarn unit
+    - name: yarn test-clients
+      run: xvfb-run --auto-servernum yarn test-clients
+    - name: yarn test-bundle
+      run: xvfb-run --auto-servernum yarn test-bundle
+    - name: yarn test-docs
+      run: xvfb-run --auto-servernum yarn test-docs
 
     - run: yarn test-lantern
     - run: yarn test-legacy-javascript
     - run: yarn i18n:checks
     - run: yarn dogfood-lhci
 
+    - name: Upload dist/
+      uses: actions/upload-artifact@v2
+      with:
+        name: dist
+        path: dist/
+
     # buildtracker runs `git merge-base HEAD origin/master` which needs more history than depth=1. https://github.com/paularmstrong/build-tracker/issues/106
     - name: Deepen git fetch (for buildtracker)
       run: git fetch --deepen=100
@@ -83,3 +75,28 @@ jobs:
 
     # Fail if any changes were written to source files (ex, from: build/build-cdt-lib.js).
     - run: git diff --exit-code
+
+  smoke:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        invert: [false, true]
+      # e.g. if smoke 0 fails, continue with smoke 1 anyway
+      fail-fast: false
+    env:
+      # The smokehouse tests run by job smoke 0. smoke-1 will run the rest.
+      SMOKE_GROUP_1: a11y oopif pwa pwa2 pwa3 dbw redirects errors offline
+    name: smoke ${{ strategy.job-index }}
+
+    steps:
+    - name: git clone
+      uses: actions/checkout@v2
+
+    - name: Use Node.js 10.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 10.x
+
+    - run: yarn --frozen-lockfile
+    - name: Run smoke tests
+      run: xvfb-run --auto-servernum yarn smoke --debug -j=1 --retries=2 --invert ${{ matrix.invert }} $SMOKE_GROUP_1
diff --git a/lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js b/lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js
@@ -34,18 +34,19 @@ const runnerPaths = {
  * Determine batches of smoketests to run, based on the `requestedIds`.
  * @param {Array<Smokehouse.TestDfn>} allTestDefns
  * @param {Array<string>} requestedIds
+ * @param {{invert: boolean}} options
  * @return {Array<Smokehouse.TestDfn>}
  */
-function getDefinitionsToRun(allTestDefns, requestedIds) {
+function getDefinitionsToRun(allTestDefns, requestedIds, {invert}) {
   let smokes = [];
   const usage = `    ${log.dim}yarn smoke ${allTestDefns.map(t => t.id).join(' ')}${log.reset}\n`;
 
-  if (requestedIds.length === 0) {
+  if (requestedIds.length === 0 && !invert) {
     smokes = [...allTestDefns];
     console.log('Running ALL smoketests. Equivalent to:');
     console.log(usage);
   } else {
-    smokes = allTestDefns.filter(test => requestedIds.includes(test.id));
+    smokes = allTestDefns.filter(test => invert !== requestedIds.includes(test.id));
     console.log(`Running ONLY smoketests for: ${smokes.map(t => t.id).join(' ')}\n`);
   }
 
@@ -57,6 +58,10 @@ function getDefinitionsToRun(allTestDefns, requestedIds) {
     console.log(usage);
   }
 
+  if (!smokes.length) {
+    throw new Error('no smoketest found to run');
+  }
+
   return smokes;
 }
 
@@ -68,14 +73,16 @@ async function begin() {
     .help('help')
     .usage('node $0 [<options>] <test-ids>')
     .example('node $0 -j=1 pwa seo', 'run pwa and seo tests serially')
+    .example('node $0 --invert byte', 'run all smoke tests but `byte`')
     .describe({
       'debug': 'Save test artifacts and output verbose logs',
       'jobs': 'Manually set the number of jobs to run at once. `1` runs all tests serially',
       'retries': 'The number of times to retry failing tests before accepting. Defaults to 0',
       'runner': 'The method of running Lighthouse',
       'tests-path': 'The path to a set of test definitions to run. Defaults to core smoke tests.',
+      'invert': 'Run all available tests except the ones provided',
     })
-    .boolean(['debug'])
+    .boolean(['debug', 'invert'])
     .alias({
       'jobs': 'j',
     })
@@ -99,7 +106,8 @@ async function begin() {
   testDefnPath = path.resolve(process.cwd(), testDefnPath);
   const requestedTestIds = argv._;
   const allTestDefns = require(testDefnPath);
-  const testDefns = getDefinitionsToRun(allTestDefns, requestedTestIds);
+  const invert = argv.invert;
+  const testDefns = getDefinitionsToRun(allTestDefns, requestedTestIds, {invert});
 
   const options = {jobs, retries, isDebug: argv.debug, lighthouseRunner};
 

diff --git a/lighthouse-cli/test/smokehouse/lighthouse-runners/bundle.js b/lighthouse-cli/test/smokehouse/lighthouse-runners/bundle.js
@@ -15,6 +15,7 @@ const ChromeLauncher = require('chrome-launcher');
 const ChromeProtocol = require('../../../../lighthouse-core/gather/connections/cri.js');
 
 // Load bundle, which creates a `global.runBundledLighthouse`.
+// @ts-ignore - file won't exist until `yarn build-all`, but not used for types anyways.
 require('../../../../dist/lighthouse-dt-bundle.js');
 
 /** @type {import('../../../../lighthouse-core/index.js')} */