Switch to Playwright for visual testing

Following the lead of Element Web, I propose we switch to using Playwright for visual testing, rather than Percy. (Context: Percy has been non-functional on this repository for months since we kept going over our billing limits.) This means that screenshots will now be stored directly in this repository as Playwright snapshots, which should be a much more cost-effective solution.

Note that this change also includes a couple of fixes for broken stories that the tests caught (yay!) The "Kitchen Sink" form stories were crashing due to lack of a TooltipProvider, and the search component was rendering inconsistently on dev vs. production builds due to bits of CSS being emitted in a different order.

.github/workflows/analysis.yml

@@ -0,0 +1,80 @@
+name: Analysis
+on:
+  pull_request: {}
+  push:
+    branches: [develop, main]
+jobs:
+  lint:
+    name: "Lint"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          cache: "yarn"
+      - name: Install deps
+        run: yarn install --frozen-lockfile
+      - name: Lint
+        run: "yarn run lint"
+      - name: Check formatting
+        run: "yarn run prettier:check"
+  test:
+    name: "Unit test"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          cache: "yarn"
+      - name: Install deps
+        run: yarn install --frozen-lockfile
+      - name: Test
+        run: "yarn run test --coverage"
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage
+          path: |
+            coverage
+            !coverage/lcov-report
+      - name: SonarCloud Scan
+        uses: SonarSource/sonarcloud-github-action@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
+  e2e:
+    name: "Test end-to-end"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          cache: "yarn"
+      - name: Install deps
+        run: yarn install --frozen-lockfile
+      - name: Build Storybook
+        run: yarn build-storybook
+      - name: Get Playwright version
+        run: echo "PLAYWRIGHT_VERSION=$(yarn info @playwright/test -A --json | jq -r .data.version)" >> $GITHUB_ENV
+      - name: Cache Playwright binaries
+        uses: actions/cache@v3
+        id: playwright-cache
+        with:
+          path: |
+            ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}
+      - name: Install Playwright binaries
+        if: steps.playwright-cache.outputs.cache-hit != 'true'
+        run: yarn playwright install --with-deps
+      - name: Run Playwright tests
+        # We use this action to get a virtual frame buffer for the browsers
+        uses: coactions/setup-xvfb@b6b4fcfb9f5a895edadc3bc76318fae0ac17c8b3 # v1
+        with:
+          run: yarn playwright test -j 100%
+      - name: Upload Playwright report
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-report--attempt-${{ github.run_attempt }}
+          path: playwright-report
+          retention-days: 14

.gitignore

@@ -10,3 +10,6 @@ dist
 yarn-error.log
 coverage
 yarn-error.log
+
+/playwright-report
+/test-results

.serverc.json

@@ -0,0 +1,3 @@
+{
+  "cleanUrls": false
+}

README.md

@@ -3,22 +3,30 @@
 # Compound Web
 
 [![](https://img.shields.io/badge/-Storybook-ff4785?logo=Storybook&logoColor=white&style=flat-square)](https://vector-im.github.io/compound-web/) [![](https://img.shields.io/github/license/vector-im/compound)](https://github.com/vector-im/compound/blob/main/LICENSE)
-[![This project is using Percy.io for visual regression testing.](https://percy.io/static/images/percy-badge.svg)](https://percy.io/c8fecada/compound-web)
 [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=vector-im_compound-web&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=vector-im_compound-web)
 [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=vector-im_compound-web&metric=coverage)](https://sonarcloud.io/summary/new_code?id=vector-im_compound-web)
 
 React implementation of Compound – Element's design system – See full documentation on https://compound.element.io and the [Figma component library](https://www.figma.com/file/rTaQE2nIUSLav4Tg3nozq7/Compound-Web-Components?type=design&node-id=129%3A4461&t=0cvCO0bpqRPGgkwa-1)
 
-## Commands
+## Development
 
-| Command                    | Runs                                 |
-| -------------------------- | ------------------------------------ |
-| `yarn dev`                 | Runs a local development environment |
-| `yarn test`                | Tests all components                 |
-| `yarn lint`                | Lints all components                 |
-| `yarn gen:component $name` | Bootstraps a new component           |
+| Command                    | Runs                          |
+| -------------------------- | ----------------------------- |
+| `yarn dev`                 | Runs a local Storybook server |
+| `yarn lint`                | Lints all components          |
+| `yarn gen:component $name` | Bootstraps a new component    |
 
-## Development
+### Testing
+
+| Command       | Runs                              |
+| ------------- | --------------------------------- |
+| `yarn test`   | Runs unit tests                   |
+| `yarn e2e`    | Runs end-to-end tests             |
+| `yarn e2e -u` | Updates end-to-end test snapshots |
+
+All components are expected to come with comprehensive unit tests and visual tests. We use Playwright to run visual tests on every story present in Storybook, so story coverage is really important! It helps us validate component implementations against the designs and prevents visual regressions at the same time.
+
+### Linking
 
 If you want to work on Compound Web as a linked package within a larger React application, TypeScript might complain about there being multiple copies of @types/react in the tree. You can work around this by linking Compound Web's copy of @types/react to your application's copy:
 
@@ -29,7 +37,7 @@ $ cd ../../../../compound-web
 $ yarn link @types/react
 ```
 
-### Release
+## Release
 
 To release a new version of Compound Web:
 

package.json

@@ -26,9 +26,7 @@
     "lint:ts": "tsc --noEmit -p .",
     "lint:styles": "npx stylelint 'src/**/*.css'",
     "test": "vitest",
-    "prepercy": "node scripts/disableStoryStore7.js && yarn build-storybook",
-    "percy": "percy storybook ./storybook-static",
-    "postpercy": "git checkout .storybook/main.ts",
+    "e2e": "playwright test",
     "build-storybook": "storybook build",
     "storybook": "storybook dev -p 6006",
     "gen:component": "node scripts/genComponent.js"
@@ -52,8 +50,7 @@
   "devDependencies": {
     "@fontsource/inconsolata": "^5.0.8",
     "@fontsource/inter": "^5.0.8",
-    "@percy/cli": "^1.27.1",
-    "@percy/storybook": "^4.3.6",
+    "@playwright/test": "^1.41.1",
     "@storybook/addon-a11y": "^7.5.1",
     "@storybook/addon-designs": "7.0.5",
     "@storybook/addon-essentials": "7.4.2",
@@ -89,6 +86,7 @@
     "react-dom": "^18.2.0",
     "resize-observer-polyfill": "^1.5.1",
     "rimraf": "^5.0.1",
+    "serve": "^14.2.1",
     "storybook": "^7.4.2",
     "stylelint": "^15.10.3",
     "stylelint-config-standard": "^34.0.0",

playwright.config.ts

@@ -0,0 +1,42 @@
+/*
+Copyright 2024 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { defineConfig, devices } from "@playwright/test";
+
+const port = 6006;
+const baseUrl = `http://localhost:${port}`;
+
+export default defineConfig({
+  testDir: "playwright",
+  use: {
+    baseURL: baseUrl,
+  },
+  projects: [
+    {
+      name: "chromium",
+      use: {
+        ...devices["Desktop Chrome"],
+        viewport: { width: 720, height: 720 },
+      },
+    },
+  ],
+  webServer: {
+    command: `npx serve -c ../.serverc.json -p ${port} -L storybook-static/`,
+    url: baseUrl,
+    reuseExistingServer: !process.env.CI,
+  },
+  reporter: "html",
+});

playwright/visual.test.ts

@@ -0,0 +1,60 @@
+/*
+Copyright 2024 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { expect, test } from "@playwright/test";
+import fs from "fs";
+
+interface Story {
+  id: string;
+  title: string;
+  name: string;
+  importPath: string;
+  tags: string[];
+}
+
+interface Stories {
+  [id: string]: Story;
+}
+
+test.describe.configure({ mode: "parallel" });
+
+const storiesPath = new URL(
+  "../storybook-static/stories.json",
+  import.meta.url,
+);
+if (!fs.existsSync(storiesPath)) {
+  console.error(
+    "Storybook manifest not found, please rebuild with 'yarn build-storybook'",
+  );
+  process.exit(1);
+}
+
+const stories = JSON.parse(fs.readFileSync(storiesPath, "utf8"))
+  .stories as Stories;
+
+// Perform visual testing on each story
+for (const story of Object.values(stories)) {
+  // Ignore things that are not stories (e.g. doc pages)
+  if (story.tags.includes("story")) {
+    test(`${story.title} ${story.name}`, async ({ page }) => {
+      const search = new URLSearchParams({ viewMode: "story", id: story.id });
+      await page.goto(`iframe.html?${search.toString()}`, {
+        waitUntil: "networkidle",
+      });
+      await expect(page).toHaveScreenshot({ fullPage: true });
+    });
+  }
+}