tests: parallelize all the tests

.github/workflows/ci.yml

@@ -7,13 +7,13 @@ name: 💡🏠
 on: [pull_request]
 
 jobs:
-  ci:
-
+  # `basics` includes all non-smoke CI, except for unit and proto
+  basics:
     runs-on: ubuntu-latest
-    strategy:
-      # e.g. if lint fails, continue to the unit tests anyway
-      fail-fast: false
 
+    # A few steps are duplicated across all jobs. Can be done better when this feature lands:
+    #   https://github.community/t/reusing-sharing-inheriting-steps-between-jobs-declarations/16851
+    #   https://github.com/actions/runner/issues/438
     steps:
     - name: git clone
       uses: actions/checkout@v2
@@ -23,63 +23,21 @@ jobs:
       with:
         node-version: 10.x
 
-    - name: Setup protoc
-      uses: arduino/setup-protoc@7ad700d
-      with:
-        version: '3.7.1'
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Set up Python
-      uses: actions/setup-python@v1
-      with:
-        python-version: 2.7
-    - name: Install Python dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install protobuf==3.7.1
-
-    # Chrome Stable is already installed by default.
-    - name: Install Chrome ToT
-      working-directory: /home/runner
-      run: bash $GITHUB_WORKSPACE/lighthouse-core/scripts/download-chrome.sh && mv chrome-linux chrome-linux-tot
-      env:
-        CHROME_PATH: /home/runner/chrome-linux-tot
-
-    # Cache yarn deps. thx https://github.com/actions/cache/blob/master/examples.md#node---yarn
-    - name: Get yarn cache directory path
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - name: Set up node_modules cache
-      uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-
     - run: yarn --frozen-lockfile
     - run: yarn build-all
-    - run: yarn diff:sample-json
-    - run: yarn type-check
-    - run: yarn lint
-    - run: yarn test-proto # Run before unit-core because the roundtrip json is needed for proto tests.
 
     # Run tests that require headfull Chrome.
     - run: sudo apt-get install xvfb
-    - run: xvfb-run --auto-servernum yarn unit
-    - run: xvfb-run --auto-servernum yarn test-clients
-    - run: xvfb-run --auto-servernum yarn test-bundle
-    - run: xvfb-run --auto-servernum yarn test-docs
-
-    - name: Smoke (ToT)
-      run: xvfb-run --auto-servernum yarn smoke --debug -j=1 --retries=2
-      env:
-        CHROME_PATH: /home/runner/chrome-linux-tot/chrome
-    - name: Smoke (stable)
-      run: xvfb-run --auto-servernum yarn smoke --debug -j=1 --retries=2
-
+    - name: yarn test-clients
+      run: xvfb-run --auto-servernum yarn test-clients
+    - name: yarn test-bundle
+      run: xvfb-run --auto-servernum yarn test-bundle
+    - name: yarn test-docs
+      run: xvfb-run --auto-servernum yarn test-docs
 
+    - run: yarn diff:sample-json
+    - run: yarn type-check
+    - run: yarn lint
     - run: yarn test-lantern
     - run: yarn test-legacy-javascript
     - run: yarn i18n:checks
@@ -95,5 +53,89 @@ jobs:
         # https://buildtracker.dev/docs/guides/github-actions#configuration
         BT_API_AUTH_TOKEN: ${{ secrets.BT_API_AUTH_TOKEN }}
 
+    - name: Upload dist
+      uses: actions/upload-artifact@v1
+      with:
+        name: dist
+        path: dist/
+
     # Fail if any changes were written to source files (ex, from: build/build-cdt-lib.js).
     - run: git diff --exit-code
+
+  # `unit` includes just unit and proto tests.
+  unit:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: git clone
+      uses: actions/checkout@v2
+
+    - name: Use Node.js 10.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 10.x
+
+    - name: Set up protoc
+      uses: arduino/setup-protoc@7ad700d
+      with:
+        version: '3.7.1'
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 2.7
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install protobuf==3.7.1
+
+    - run: yarn --frozen-lockfile
+
+    - run: yarn test-proto # Run before unit-core because the roundtrip json is needed for proto tests.
+
+    - run: sudo apt-get install xvfb
+    - name: yarn unit
+      run: xvfb-run --auto-servernum yarn unit
+
+  # `smoke` runs as a matrix across 4 jobs:
+  #  * The smoketest groups are split across two runners, to parallelize.
+  #  * Then, those are run with both Chrome stable and ToT Chromium, in parallel
+  smoke:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        chrome-channel: ['stable', 'ToT']
+        smoke-test-invert: [false, true]
+      # e.g. if smoke 0 fails, continue with smoke 1 anyway
+      fail-fast: false
+    env:
+      # The smokehouse tests run by job `smoke_0`. `smoke_1` will run the rest.
+      SMOKE_GROUP_1: a11y oopif pwa pwa2 pwa3 dbw redirects errors offline
+    name: smoke_${{ strategy.job-index }}_${{ matrix.chrome-channel }}
+
+    steps:
+    - name: git clone
+      uses: actions/checkout@v2
+
+    - name: Use Node.js 10.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 10.x
+
+    - name: Define ToT chrome path
+      if: matrix.chrome-channel == 'ToT'
+      run: echo "::set-env name=CHROME_PATH::/home/runner/chrome-linux-tot/chrome"
+
+    # Chrome Stable is already installed by default.
+    - name: Install Chrome ToT
+      if: matrix.chrome-channel == 'ToT'
+      working-directory: /home/runner
+      run: bash $GITHUB_WORKSPACE/lighthouse-core/scripts/download-chrome.sh && mv chrome-linux chrome-linux-tot
+
+    - run: yarn --frozen-lockfile
+
+    - run: sudo apt-get install xvfb
+
+    - name: Run smoke tests
+      run: xvfb-run --auto-servernum yarn smoke --debug -j=1 --retries=2 --invert-match ${{ matrix.smoke-test-invert }} $SMOKE_GROUP_1

lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js

@@ -34,18 +34,19 @@ const runnerPaths = {
  * Determine batches of smoketests to run, based on the `requestedIds`.
  * @param {Array<Smokehouse.TestDfn>} allTestDefns
  * @param {Array<string>} requestedIds
+ * @param {{invertMatch: boolean}} options
  * @return {Array<Smokehouse.TestDfn>}
  */
-function getDefinitionsToRun(allTestDefns, requestedIds) {
+function getDefinitionsToRun(allTestDefns, requestedIds, {invertMatch}) {
   let smokes = [];
   const usage = `    ${log.dim}yarn smoke ${allTestDefns.map(t => t.id).join(' ')}${log.reset}\n`;
 
-  if (requestedIds.length === 0) {
+  if (requestedIds.length === 0 && !invertMatch) {
     smokes = [...allTestDefns];
     console.log('Running ALL smoketests. Equivalent to:');
     console.log(usage);
   } else {
-    smokes = allTestDefns.filter(test => requestedIds.includes(test.id));
+    smokes = allTestDefns.filter(test => invertMatch !== requestedIds.includes(test.id));
     console.log(`Running ONLY smoketests for: ${smokes.map(t => t.id).join(' ')}\n`);
   }
 
@@ -57,6 +58,10 @@ function getDefinitionsToRun(allTestDefns, requestedIds) {
     console.log(usage);
   }
 
+  if (!smokes.length) {
+    throw new Error('no smoketest found to run');
+  }
+
   return smokes;
 }
 
@@ -68,14 +73,16 @@ async function begin() {
     .help('help')
     .usage('node $0 [<options>] <test-ids>')
     .example('node $0 -j=1 pwa seo', 'run pwa and seo tests serially')
+    .example('node $0 --invert-match byte', 'run all smoke tests but `byte`')
     .describe({
       'debug': 'Save test artifacts and output verbose logs',
       'jobs': 'Manually set the number of jobs to run at once. `1` runs all tests serially',
       'retries': 'The number of times to retry failing tests before accepting. Defaults to 0',
       'runner': 'The method of running Lighthouse',
       'tests-path': 'The path to a set of test definitions to run. Defaults to core smoke tests.',
+      'invert-match': 'Run all available tests except the ones provided',
     })
-    .boolean(['debug'])
+    .boolean(['debug', 'invert-match'])
     .alias({
       'jobs': 'j',
     })
@@ -99,7 +106,8 @@ async function begin() {
   testDefnPath = path.resolve(process.cwd(), testDefnPath);
   const requestedTestIds = argv._;
   const allTestDefns = require(testDefnPath);
-  const testDefns = getDefinitionsToRun(allTestDefns, requestedTestIds);
+  const invertMatch = argv.invertMatch;
+  const testDefns = getDefinitionsToRun(allTestDefns, requestedTestIds, {invertMatch});
 
   const options = {jobs, retries, isDebug: argv.debug, lighthouseRunner};
 

lighthouse-cli/test/smokehouse/lighthouse-runners/bundle.js

@@ -15,7 +15,8 @@ const ChromeLauncher = require('chrome-launcher');
 const ChromeProtocol = require('../../../../lighthouse-core/gather/connections/cri.js');
 
 // Load bundle, which creates a `global.runBundledLighthouse`.
-require('../../../../dist/lighthouse-dt-bundle.js');
+// @ts-ignore - file won't exist until `yarn build-all`, but not used for types anyways.
+require('../../../../dist/lighthouse-dt-bundle.js'); // eslint-disable-line
 
 /** @type {import('../../../../lighthouse-core/index.js')} */
 // @ts-ignore - not worth giving test global an actual type.