From 1ca18c459a191e81d7f1f5f9ed18d6ce015aed16 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 2 Oct 2024 12:12:28 -0400 Subject: [PATCH] Introduce WebkitHeadless browser testing (#8491) * Introduce WebkitHeadless browser testing Replace Safari with WebkitHeadless in list of supported karma browsers Install playwright browsers in CI Set WEBKIT_HEADLESS_BIN Update launcher version Run firestore test changed on push Fix Don't use in test-changed-firestore Ignore failing tests * Formatting * Respond to review comments --- .github/workflows/test-changed-auth.yml | 25 +++++++++ .github/workflows/test-changed-firestore.yml | 58 +++++++++++++++++++- .github/workflows/test-changed.yml | 26 +++++++++ config/karma.base.js | 36 ++++++++++-- package.json | 3 +- scripts/ci-test/testConfig.ts | 14 +++++ yarn.lock | 42 ++++++++++---- 7 files changed, 188 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-changed-auth.yml b/.github/workflows/test-changed-auth.yml index 67d16551b3e..52b08c12e21 100644 --- a/.github/workflows/test-changed-auth.yml +++ b/.github/workflows/test-changed-auth.yml @@ -102,3 +102,28 @@ jobs: run: xvfb-run yarn test:changed auth env: BROWSERS: 'Firefox' + + test-webkit: + name: Test Auth on Webkit if Changed + runs-on: macos-latest + + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Node (20) + uses: actions/setup-node@v3 + with: + node-version: 20.x + - name: Test setup and yarn install + run: | + cp config/ci.config.json config/project.json + yarn + npx playwright install webkit + - name: build + run: yarn build:changed auth + - name: Run tests on changed packages + run: yarn test:changed auth + env: + BROWSERS: 'WebkitHeadless' \ No newline at end of file diff --git a/.github/workflows/test-changed-firestore.yml b/.github/workflows/test-changed-firestore.yml index e148d164909..4a924d14e25 100644 --- a/.github/workflows/test-changed-firestore.yml +++ b/.github/workflows/test-changed-firestore.yml @@ -231,6 +231,62 @@ jobs: BROWSERS: 'Firefox' EXPERIMENTAL_MODE: true + compat-test-webkit: + name: Test Firestore Compatible on Webkit + runs-on: macos-latest + needs: build + if: ${{ needs.build.outputs.changed == 'true'}} + steps: + - name: Set up Node (20) + uses: actions/setup-node@v3 + with: + node-version: 20.x + - name: Download build archive + uses: actions/download-artifact@v3 + with: + name: build.tar.gz + - name: Unzip build artifact + run: tar xf build.tar.gz + - name: Test setup + run: | + cp config/ci.config.json config/project.json + npx playwright install webkit + - name: Run compat tests + run: cd packages/firestore-compat && yarn run test:ci + env: + BROWSERS: 'WebkitHeadless' + + test-webkit: + name: Test Firestore on Webkit + strategy: + matrix: + # TODO (dlarocque): Add test:travis once the browser tests are isolated + # Exclude test:travis for now, since it includes node tests, which are failing for + # some reason. + test-name: ["test:browser", "test:lite:browser", "test:browser:prod:nameddb", "test:lite:browser:nameddb"] + runs-on: macos-latest + needs: build + if: ${{ needs.build.outputs.changed == 'true'}} + steps: + - name: Download build archive + uses: actions/download-artifact@v3 + with: + name: build.tar.gz + - name: Unzip build artifact + run: tar xf build.tar.gz + - name: Set up Node (20) + uses: actions/setup-node@v3 + with: + node-version: 20.x + - name: Test setup + run: | + cp config/ci.config.json config/project.json + npx playwright install webkit + - name: Run tests + run: cd packages/firestore && yarn run ${{ matrix.test-name }} + env: + BROWSERS: 'WebkitHeadless' + EXPERIMENTAL_MODE: true # A job that fails if any required job in the test matrix fails, # to be used as a required check for merging. check-required-tests: @@ -241,4 +297,4 @@ jobs: steps: - name: Check test matrix if: needs.build.result == 'failure' || needs.test-chrome.result == 'failure' || needs.compat-test-chrome.result == 'failure' - run: exit 1 + run: exit 1 \ No newline at end of file diff --git a/.github/workflows/test-changed.yml b/.github/workflows/test-changed.yml index 7da82df4a26..5119c83cb37 100644 --- a/.github/workflows/test-changed.yml +++ b/.github/workflows/test-changed.yml @@ -78,3 +78,29 @@ jobs: run: xvfb-run yarn test:changed core env: BROWSERS: 'Firefox' + + + test-webkit: + name: Test Packages With Changed Files in Webkit + runs-on: macos-latest + + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Node (20) + uses: actions/setup-node@v3 + with: + node-version: 20.x + - name: Test setup and yarn install + run: | + cp config/ci.config.json config/project.json + yarn + npx playwright install webkit + - name: build + run: yarn build:changed core + - name: Run tests on changed packages + run: yarn test:changed core + env: + BROWSERS: 'WebkitHeadless' \ No newline at end of file diff --git a/config/karma.base.js b/config/karma.base.js index e074987f29b..49824296740 100644 --- a/config/karma.base.js +++ b/config/karma.base.js @@ -20,6 +20,33 @@ const path = require('path'); const webpackTestConfig = require('./webpack.test'); const { argv } = require('yargs'); +function determineBrowsers() { + const supportedBrowsers = ['ChromeHeadless', 'WebkitHeadless', 'Firefox']; + + if (process.env.BROWSERS) { + const browsers = process.env.BROWSERS.split(','); + + const validBrowsers = browsers.filter(browser => + supportedBrowsers.includes(browser) + ); + if (validBrowsers.length === 0) { + console.error( + `The \'BROWSER\' environment variable was set, but no supported browsers were listed. The supported browsers are ${JSON.stringify( + supportedBrowsers + )}.` + ); + return []; + } else { + return validBrowsers; + } + } else { + console.log( + "The 'BROWSER' environment variable is undefined. Defaulting to 'ChromeHeadless'." + ); + return ['ChromeHeadless']; + } +} + const config = { // disable watcher autoWatch: false, @@ -57,10 +84,11 @@ const config = { // changes autoWatch: false, - // start these browsers - // available browser launchers: - // https://npmjs.org/browse/keyword/karma-launcher - browsers: process.env?.BROWSERS?.split(',') ?? ['ChromeHeadless'], + // Browsers to launch for testing + // To use a custom set of browsers, define the BROWSERS environment variable as a comma-seperated list. + // Supported browsers are 'ChromeHeadless', 'WebkitHeadless', and 'Firefox'. + // See: https://karma-runner.github.io/6.4/config/browsers.html + browsers: determineBrowsers(), webpack: webpackTestConfig, diff --git a/package.json b/package.json index 7c65287bb20..c301feb69de 100644 --- a/package.json +++ b/package.json @@ -120,10 +120,10 @@ "karma-firefox-launcher": "2.1.3", "karma-mocha": "2.0.1", "karma-mocha-reporter": "2.2.5", - "karma-safari-launcher": "1.0.0", "karma-sourcemap-loader": "0.4.0", "karma-spec-reporter": "0.0.36", "karma-summary-reporter": "3.1.1", + "karma-webkit-launcher": "2.6.0", "karma-webpack": "5.0.0", "lcov-result-merger": "3.3.0", "lerna": "4.0.0", @@ -139,6 +139,7 @@ "nyc": "15.1.0", "ora": "5.4.1", "patch-package": "7.0.2", + "playwright": "1.46.1", "postinstall-postinstall": "2.1.0", "prettier": "2.8.7", "protractor": "5.4.2", diff --git a/scripts/ci-test/testConfig.ts b/scripts/ci-test/testConfig.ts index 5f458a07605..0202c699154 100644 --- a/scripts/ci-test/testConfig.ts +++ b/scripts/ci-test/testConfig.ts @@ -28,11 +28,25 @@ export interface TestConfig { alwaysIncludePackages?: string[]; } +// These tests are flaky on WebkitHeadless for some reason, so skip them. +// TODO (dlarocque): Fix the flakes and remove this +const ignoredWebkitCoreTests = process.env?.BROWSERS?.includes('WebkitHeadless') + ? [ + '@firebase/app-check', + '@firebase/installations', + '@firebase/storage', + '@firebase/storage-compat', + '@firebase/database', + '@firebase/database-compat' + ] + : []; + export const testConfig: { [key: string]: TestConfig | undefined; } = { 'core': { 'ignorePackages': [ + ...ignoredWebkitCoreTests, '@firebase/firestore', '@firebase/firestore-compat', 'firebase-firestore-integration-test', diff --git a/yarn.lock b/yarn.lock index 92372aa94a3..ddc8001a752 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8653,6 +8653,11 @@ fs.realpath@^1.0.0: resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= +fsevents@2.3.2, fsevents@^2.3.2, fsevents@~2.3.2: + version "2.3.2" + resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== + fsevents@^1.2.7: version "1.2.13" resolved "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz" @@ -8661,11 +8666,6 @@ fsevents@^1.2.7: bindings "^1.5.0" nan "^2.12.1" -fsevents@^2.3.2, fsevents@~2.3.2: - version "2.3.2" - resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz" - integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== - ftp@^0.3.10: version "0.3.10" resolved "https://registry.npmjs.org/ftp/-/ftp-0.3.10.tgz" @@ -11339,11 +11339,6 @@ karma-mocha@2.0.1: dependencies: minimist "^1.2.3" -karma-safari-launcher@1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/karma-safari-launcher/-/karma-safari-launcher-1.0.0.tgz" - integrity sha1-lpgqLMR9BmquccVTursoMZEVos4= - karma-sourcemap-loader@0.4.0: version "0.4.0" resolved "https://registry.npmjs.org/karma-sourcemap-loader/-/karma-sourcemap-loader-0.4.0.tgz" @@ -11414,6 +11409,14 @@ karma-typescript@5.5.4: util "^0.12.1" vm-browserify "^1.1.2" +karma-webkit-launcher@2.6.0: + version "2.6.0" + resolved "https://registry.npmjs.org/karma-webkit-launcher/-/karma-webkit-launcher-2.6.0.tgz#2e3ba096b69139e608d9ce6e89816a17f358b700" + integrity sha512-IDURopxJ1SbuqnvPaE+lP2qiP2Ie7I+ojwJRBpr0tfGwObsaVdjMkUkmZ1BcXUtYRt5ogs9cyCH2Wb9sNv0BbQ== + dependencies: + is-ci "^3.0.1" + uuid "^10.0.0" + karma-webpack@5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/karma-webpack/-/karma-webpack-5.0.0.tgz" @@ -14119,6 +14122,20 @@ pkg-dir@^4.1.0, pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" +playwright-core@1.46.1: + version "1.46.1" + resolved "https://registry.npmjs.org/playwright-core/-/playwright-core-1.46.1.tgz#28f3ab35312135dda75b0c92a3e5c0e7edb9cc8b" + integrity sha512-h9LqIQaAv+CYvWzsZ+h3RsrqCStkBHlgo6/TJlFst3cOTlLghBQlJwPOZKQJTKNaD3QIB7aAVQ+gfWbN3NXB7A== + +playwright@1.46.1: + version "1.46.1" + resolved "https://registry.npmjs.org/playwright/-/playwright-1.46.1.tgz#ea562bc48373648e10420a10c16842f0b227c218" + integrity sha512-oPcr1yqoXLCkgKtD5eNUPLiN40rYEM39odNpIb6VE6S7/15gJmA1NzVv6zJYusV0e7tzvkU/utBFNa/Kpxmwng== + dependencies: + playwright-core "1.46.1" + optionalDependencies: + fsevents "2.3.2" + plugin-error@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/plugin-error/-/plugin-error-1.0.1.tgz" @@ -17657,6 +17674,11 @@ utils-merge@1.0.1: resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= +uuid@^10.0.0: + version "10.0.0" + resolved "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz#5a95aa454e6e002725c79055fd42aaba30ca6294" + integrity sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ== + uuid@^3.3.2, uuid@^3.3.3: version "3.4.0" resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz"