Run all e2e/component vrt & aat tests a second time with all css modu…

…le flags on (#5070) * Run vrt tests with all flags on * Remove inline feature flag tests * Set flags on whole job * Also run aat twice * Test failure with a module only style change * lowercase * Pass flags to VITE * Change buttonbase back * change blob path * Fix text component * Fix path to DIR * Fix avatar square * revert blob changes * Add calc to value * Fixes for ButtonBase and Timeline * test(vrt): update snapshots * Rename no flag to no flag * Fix needs * Remove feature flag check --------- Co-authored-by: jonrohan <jonrohan@users.noreply.github.com>
primer · Oct 7, 2024 · b91a5e5 · b91a5e5
1 parent c2f64aa
commit b91a5e5
Show file tree

Hide file tree

Showing 12 changed files with 1,955 additions and 2,412 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -144,7 +144,7 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
-          name: vrt-${{ matrix.shard }}
+          name: vrt-no-flag-${{ matrix.shard }}
           path: blob-report
           retention-days: 1
 
@@ -166,14 +166,92 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           path: all-blob-reports
-          pattern: vrt-*
+          pattern: vrt-no-flag-*
           merge-multiple: true
       - name: merge all reports
         run: npx playwright merge-reports --reporter html ./all-blob-reports
       - name: Upload report
         uses: actions/upload-artifact@v4
         with:
-          name: vrt
+          name: vrt-no-flag
+          path: playwright-report
+      - name: check vrt-runner job status
+        if: ${{ needs.vrt-runner.result == 'failure' }}
+        run: exit 1
+
+  vrt-runner-all-flags:
+    runs-on: ubuntu-latest-8-cores
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
+    env:
+      VITE_PRIMER_REACT_CSS_MODULES_TEAM: 1
+      VITE_PRIMER_REACT_CSS_MODULES_STAFF: 1
+      VITE_PRIMER_REACT_CSS_MODULES_GA: 1
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js 20.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: 'npm'
+      - run: npm i -g npm@^10.5.1
+      - name: Install dependencies
+        run: npm ci
+      - name: Build storybook
+        run: npx storybook build
+        working-directory: packages/react
+      - name: Run storybook
+        id: storybook
+        working-directory: packages/react
+        run: |
+          npx serve -l 6006 storybook-static &
+          pid=$!
+          echo "pid=$pid" >> $GITHUB_OUTPUT
+          sleep 5
+      - name: Run VRT
+        uses: docker://mcr.microsoft.com/playwright:v1.43.0-jammy
+        env:
+          STORYBOOK_URL: 'http://172.17.0.1:6006'
+        with:
+          args: npx playwright test --grep @vrt --shard="${{ matrix.shard }}/${{ strategy.job-total }}"
+      - name: Stop storybook
+        run: kill ${{ steps.storybook.outputs.pid }}
+      - name: Upload report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: vrt-all-flags-${{ matrix.shard }}
+          path: blob-report
+          retention-days: 1
+
+  vrt-all-flags:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: vrt-runner-all-flags
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js 20.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: 'npm'
+      - run: npm i -g npm@^10.5.1
+      - name: install dependencies
+        run: npm ci
+      - name: download all reports
+        uses: actions/download-artifact@v4
+        with:
+          path: all-blob-reports
+          pattern: vrt-all-flags-*
+          merge-multiple: true
+      - name: merge all reports
+        run: npx playwright merge-reports --reporter html ./all-blob-reports
+      - name: Upload report
+        uses: actions/upload-artifact@v4
+        with:
+          name: vrt-all-flags
           path: playwright-report
       - name: check vrt-runner job status
         if: ${{ needs.vrt-runner.result == 'failure' }}
@@ -218,7 +296,7 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
-          name: axe-${{ matrix.shard }}
+          name: axe-no-flag-${{ matrix.shard }}
           path: blob-report
           retention-days: 1
 
@@ -240,7 +318,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           path: all-blob-reports
-          pattern: axe-*
+          pattern: axe-no-flag-*
           merge-multiple: true
       - name: merge all reports
         run: npx playwright merge-reports --reporter html ./all-blob-reports
@@ -253,6 +331,84 @@ jobs:
         if: ${{ needs.aat-runner.result == 'failure' }}
         run: exit 1
 
+  aat-runner-all-flags:
+    runs-on: ubuntu-latest-8-cores
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
+    env:
+      VITE_PRIMER_REACT_CSS_MODULES_TEAM: 1
+      VITE_PRIMER_REACT_CSS_MODULES_STAFF: 1
+      VITE_PRIMER_REACT_CSS_MODULES_GA: 1
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js 20.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: 'npm'
+      - run: npm i -g npm@^10.5.1
+      - name: Install dependencies
+        run: npm ci
+      - name: Build storybook
+        run: npx storybook build
+        working-directory: packages/react
+      - name: Run storybook
+        id: storybook
+        working-directory: packages/react
+        run: |
+          npx serve -l 6006 storybook-static &
+          pid=$!
+          echo "pid=$pid" >> $GITHUB_OUTPUT
+          sleep 5
+      - name: Run AAT
+        uses: docker://mcr.microsoft.com/playwright:v1.43.0-jammy
+        env:
+          STORYBOOK_URL: 'http://172.17.0.1:6006'
+        with:
+          args: npx playwright test --grep @aat --shard="${{ matrix.shard }}/${{ strategy.job-total }}"
+      - name: Stop storybook
+        run: kill ${{ steps.storybook.outputs.pid }}
+      - name: Upload report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: axe-all-flags-${{ matrix.shard }}
+          path: blob-report
+          retention-days: 1
+
+  aat-all-flags:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: aat-runner-all-flags
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js 20.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: 'npm'
+      - run: npm i -g npm@^10.5.1
+      - name: install dependencies
+        run: npm ci
+      - name: download all reports
+        uses: actions/download-artifact@v4
+        with:
+          path: all-blob-reports
+          pattern: axe-all-flags-*
+          merge-multiple: true
+      - name: merge all reports
+        run: npx playwright merge-reports --reporter html ./all-blob-reports
+      - name: Upload report
+        uses: actions/upload-artifact@v4
+        with:
+          name: axe-all-flags
+          path: playwright-report
+      - name: Check aat-runner job status
+        if: ${{ needs.aat-runner.result == 'failure' }}
+        run: exit 1
+
   build-components-json:
     runs-on: ubuntu-latest
     steps: