Setup visual testing (#50)

* Add cloudflare job to publish pages * Try like this * Fix build path * Try end to end * Fix installation * Use var instead of secret for CLOUDFLARE_PROJECT_NAME * Use new puppeteer mode * Add more tests * Run in parallel * Go back to no-concurrency * Test pagespeed * Print output * Fix thresholds
GitbookIO · Dec 19, 2023 · 720c2ca · 720c2ca
1 parent c935700
commit 720c2ca
Show file tree

Hide file tree

Showing 7 changed files with 312 additions and 1 deletion.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -0,0 +1,72 @@
+on: [push]
+name: CI
+
+jobs:
+    deploy:
+        name: Deploy to Cloudflare Pages
+        runs-on: ubuntu-latest
+        permissions:
+            contents: read
+            deployments: write
+        outputs:
+            deployment_url: ${{ steps.cloudflare.outputs.url }}
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v3
+            - name: Setup bun
+              uses: oven-sh/setup-bun@v1
+            - name: Install dependencies
+              run: bun install
+            - name: Build Next.js with next-on-pages
+              run: bun run build:cloudflare
+            - id: cloudflare
+              name: Publish to Cloudflare Pages
+              uses: cloudflare/pages-action@v1
+              with:
+                  apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+                  accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+                  projectName: ${{ vars.CLOUDFLARE_PROJECT_NAME }}
+                  directory: ./.vercel/output/static
+                  gitHubToken: ${{ secrets.GITHUB_TOKEN }}
+                  workingDirectory: ./
+            - name: Outputs
+              run: |
+                  echo "ID: ${{ steps.cloudflare.outputs.id }}"
+                  echo "URL: ${{ steps.cloudflare.outputs.url }}"
+                  echo "Environment: ${{ steps.cloudflare.outputs.environment }}"
+                  echo "Alias: ${{ steps.cloudflare.outputs.alias }}"
+    visual-testing:
+        runs-on: ubuntu-latest
+        name: Visual Testing
+        needs: deploy
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v3
+            - name: Setup bun
+              uses: oven-sh/setup-bun@v1
+            - name: Install dependencies
+              run: bun install
+            - name: Run visual tests
+              run: bun ./tests/visual-testing.ts $DEPLOYMENT_URL
+              env:
+                  DEPLOYMENT_URL: ${{needs.deploy.outputs.deployment_url}}
+            - name: Upload to Argos
+              run: bun x argos upload --token $ARGOS_TOKEN ./screenshots
+              env:
+                  ARGOS_TOKEN: ${{ secrets.ARGOS_TOKEN }}
+    pagespeed-testing:
+        runs-on: ubuntu-latest
+        name: PageSpeed Testing
+        needs: deploy
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v3
+            - name: Setup bun
+              uses: oven-sh/setup-bun@v1
+            - name: Install dependencies
+              run: bun install
+            - name: Run visual tests
+              run: bun ./tests/pagespeed-testing.ts $DEPLOYMENT_URL
+              env:
+                  DEPLOYMENT_URL: ${{needs.deploy.outputs.deployment_url}}
+                  PAGESPEED_API_KEY: ${{ secrets.PAGESPEED_API_KEY }}
diff --git a/bun.lockb b/bun.lockb
diff --git a/package.json b/package.json
@@ -42,12 +42,15 @@
         "tailwind-shades": "^1.1.2"
     },
     "devDependencies": {
+        "@argos-ci/cli": "^1.0.4",
+        "@argos-ci/puppeteer": "^1.2.1",
         "@cloudflare/next-on-pages": "^1.7.3",
         "@types/js-cookie": "^3.0.6",
         "@types/jsontoxml": "^1.0.5",
         "@types/katex": "^0.16.5",
         "@types/node": "^20",
         "@types/parse-cache-control": "^1.0.4",
+        "@types/psi": "^4.1.6",
         "@types/react": "^18",
         "@types/react-dom": "^18",
         "autoprefixer": "^10",
@@ -56,6 +59,8 @@
         "eslint-plugin-import": "^2.29.0",
         "postcss": "^8",
         "prettier": "^3.0.3",
+        "psi": "^4.1.0",
+        "puppeteer": "^21.6.1",
         "tailwindcss": "^3.3.4",
         "typescript": "^5"
     }

diff --git a/tests/pagespeed-testing.ts b/tests/pagespeed-testing.ts
@@ -0,0 +1,37 @@
+import psi from 'psi';
+import { getContentTestURL, getTargetURL } from './utils';
+
+interface Test {
+    url: string;
+    strategy: 'mobile' | 'desktop';
+    threshold: number;
+}
+
+const tests: Array<Test> = [
+    {
+        url: 'https://docs.gitbook.com',
+        strategy: 'desktop',
+        threshold: 90,
+    },
+
+    {
+        url: 'https://docs.gitbook.com',
+        strategy: 'mobile',
+        threshold: 80,
+    },
+];
+
+console.log(`Starting PageSpeed testing with ${getTargetURL()}...`);
+
+for (const test of tests) {
+    const url = getContentTestURL(test.url);
+
+    console.log(`Testing ${url} on ${test.strategy}...`);
+    await psi.output(url, {
+        strategy: test.strategy,
+        threshold: test.threshold,
+        key: process.env.PAGESPEED_API_KEY,
+    });
+
+    console.log('');
+}
diff --git a/tests/utils.ts b/tests/utils.ts
@@ -0,0 +1,25 @@
+/**
+ * Get the target URL from the command line arguments.
+ */
+export function getTargetURL() {
+    const targetUrl = Bun.argv[2];
+    if (!targetUrl) {
+        console.error('No target URL provided');
+        process.exit(1);
+    }
+
+    return targetUrl;
+}
+
+/**
+ * Get the URL to load for a content
+ */
+export function getContentTestURL(input: string): string {
+    const url = new URL(getTargetURL());
+    const contentUrl = new URL(input);
+
+    url.pathname = `${contentUrl.host}${contentUrl.pathname}`;
+    url.search = contentUrl.search;
+
+    return url.toString();
+}
diff --git a/tests/visual-testing.ts b/tests/visual-testing.ts
@@ -0,0 +1,172 @@
+import puppeteer from 'puppeteer';
+import { argosScreenshot } from '@argos-ci/puppeteer';
+import { getContentTestURL, getTargetURL } from './utils';
+
+interface Test {
+    name: string;
+    url: string;
+}
+
+interface TestsCase {
+    name: string;
+    baseUrl: string;
+    tests: Array<Test>;
+}
+
+const testCases: TestsCase[] = [
+    {
+        name: 'GitBook',
+        baseUrl: 'https://docs.gitbook.com',
+        tests: [
+            {
+                name: 'Home',
+                url: '',
+            },
+            {
+                name: 'Search',
+                url: '?q=',
+            },
+            {
+                name: 'Search Results',
+                url: '?q=gitsync',
+            },
+            {
+                name: 'AI Search',
+                url: '?q=What+is+GitBook%3F&ask=1',
+            },
+        ],
+    },
+    {
+        name: 'Snyk',
+        baseUrl: 'https://docs.snyk.io',
+        tests: [
+            {
+                name: 'Home',
+                url: '',
+            },
+        ],
+    },
+    {
+        name: 'Rocket.Chat',
+        baseUrl: 'https://docs.rocket.chat',
+        tests: [
+            {
+                name: 'Home',
+                url: '',
+            },
+        ],
+    },
+    {
+        name: 'Commerce Layer',
+        baseUrl: 'https://docs.commercelayer.io/core/',
+        tests: [
+            {
+                name: 'Home',
+                url: '',
+            },
+            {
+                name: 'API Reference',
+                url: 'v/api-reference/',
+            },
+        ],
+    },
+    {
+        name: 'Naviga',
+        baseUrl: 'https://docs.navigaglobal.com/naviga-dashboard-overview/',
+        tests: [
+            {
+                name: 'Home',
+                url: 'v/dashboard-5.4/',
+            },
+        ],
+    },
+    {
+        name: 'Mattermost',
+        baseUrl: 'https://handbook.mattermost.com/',
+        tests: [
+            {
+                name: 'Home',
+                url: '',
+            },
+        ],
+    },
+    {
+        name: 'Tile DB',
+        baseUrl: 'https://docs.tiledb.com/main/',
+        tests: [
+            {
+                name: 'Home',
+                url: '',
+            },
+        ],
+    },
+    {
+        name: 'Castordoc',
+        baseUrl: 'https://docs.castordoc.com/',
+        tests: [
+            {
+                name: 'Home',
+                url: '',
+            },
+        ],
+    },
+    {
+        name: 'Nimbleway',
+        baseUrl: 'https://docs.nimbleway.com/',
+        tests: [
+            {
+                name: 'Home',
+                url: '',
+            },
+        ],
+    },
+    {
+        name: 'Parcellab',
+        baseUrl: 'https://how.parcellab.works/docs/',
+        tests: [
+            {
+                name: 'Home',
+                url: '',
+            },
+        ],
+    },
+    {
+        name: 'CitrusAd',
+        baseUrl: 'https://help.citrusad.com/citrus-ads/',
+        tests: [
+            {
+                name: 'Home',
+                url: '',
+            },
+        ],
+    },
+];
+
+console.log(`Starting visual testing with ${getTargetURL()}...`);
+
+const browser = await puppeteer.launch({
+    headless: 'new',
+});
+const page = await browser.newPage();
+
+for (const testCase of testCases) {
+    for (const test of testCase.tests) {
+        const contentUrl = new URL(test.url, testCase.baseUrl);
+        const url = getContentTestURL(contentUrl.toString());
+        const start = Date.now();
+
+        console.log(`Testing ${testCase.name} - ${test.name} (${url})...`);
+
+        await page.goto(url, { waitUntil: 'networkidle2' });
+
+        await argosScreenshot(page, `${testCase.name} - ${test.name}.png`, {
+            viewports: ['iphone-x', 'ipad-2', 'macbook-13'],
+        });
+        console.log(`✅ Done in ${((Date.now() - start) / 1000).toFixed(2)}s`);
+        console.log('');
+    }
+}
+await page.close();
+await browser.close();
+
+console.log('All done!');
diff --git a/tsconfig.json b/tsconfig.json
@@ -1,6 +1,6 @@
 {
     "compilerOptions": {
-        "target": "es5",
+        "target": "esnext",
         "lib": ["dom", "dom.iterable", "esnext"],
         "allowJs": true,
         "skipLibCheck": true,