dev(testing): add jest-playwright and reduce flakiness of e2e tests

.eslintrc.yaml

@@ -15,7 +15,12 @@ extends:
   - plugin:import/recommended
   - plugin:import/typescript
   - plugin:prettier/recommended
-  - prettier # Removes eslint rules that conflict with prettier.
+  # Recommended by jest-playwright
+  # https://github.com/playwright-community/jest-playwright#globals
+  - plugin:jest-playwright/recommended
+  # Prettier should always be last
+  # Removes eslint rules that conflict with prettier.
+  - prettier
 
 rules:
   # Sometimes you need to add args to implement a function signature even

.github/workflows/ci.yaml

@@ -63,7 +63,7 @@ jobs:
       - uses: microsoft/playwright-github-action@v1
       - name: Install dependencies and run end-to-end tests
         run: |
-          ./release-packages/code-server*-linux-amd64/bin/code-server &
+          ./release-packages/code-server*-linux-amd64/bin/code-server --log trace &
           yarn --frozen-lockfile
           yarn test:e2e
       - name: Upload test artifacts

ci/dev/test-e2e.sh

@@ -15,7 +15,7 @@ main() {
     echo -e "\n"
     exit 1
   fi
-  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --config ./test/jest.e2e.config.ts
+  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --config ./test/jest.e2e.config.ts --runInBand
 }
 
 main "$@"

ci/steps/test-e2e.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 main() {
   cd "$(dirname "$0")/../.."
 
-  "./release-packages/code-server*-linux-amd64/bin/code-server" --home "$CODE_SERVER_ADDRESS"/healthz &
+  "./release-packages/code-server*-linux-amd64/bin/code-server" &
   yarn --frozen-lockfile
   yarn test:e2e
 }

lib/vscode/src/vs/platform/extensionManagement/node/extensionDownloader.ts

@@ -76,7 +76,7 @@ export class ExtensionsDownloader extends Disposable {
 	private async cleanUp(): Promise<void> {
 		try {
 			if (!(await this.fileService.exists(this.extensionsDownloadDir))) {
-				this.logService.trace('Extension VSIX downlads cache dir does not exist');
+				this.logService.trace('Extension VSIX downloads cache dir does not exist');
 				return;
 			}
 			const folderStat = await this.fileService.resolve(this.extensionsDownloadDir, { resolveMetadata: true });

package.json

@@ -61,6 +61,7 @@
     "eslint-config-prettier": "^8.1.0",
     "eslint-import-resolver-alias": "^1.1.2",
     "eslint-plugin-import": "^2.18.2",
+    "eslint-plugin-jest-playwright": "^0.2.1",
     "eslint-plugin-prettier": "^3.1.0",
     "istanbul-badges-readme": "^1.2.0",
     "leaked-handles": "^5.2.0",

test/e2e/browser.test.ts

@@ -0,0 +1,35 @@
+/// <reference types="jest-playwright-preset" />
+
+// This test is for nothing more than to make sure
+// tests are running in multiple browsers
+describe("Browser gutcheck", () => {
+  beforeEach(async () => {
+    await jestPlaywright.resetBrowser({
+      logger: {
+        isEnabled: (name) => name === "browser",
+        log: (name, severity, message, args) => console.log(`${name} ${message}`),
+      },
+    })
+  })
+
+  test("should display correct browser based on userAgent", async () => {
+    const displayNames = {
+      chromium: "Chrome",
+      firefox: "Firefox",
+      webkit: "Safari",
+    }
+    const userAgent = await page.evaluate("navigator.userAgent")
+
+    if (browserName === "chromium") {
+      expect(userAgent).toContain(displayNames[browserName])
+    }
+
+    if (browserName === "firefox") {
+      expect(userAgent).toContain(displayNames[browserName])
+    }
+
+    if (browserName === "webkit") {
+      expect(userAgent).toContain(displayNames[browserName])
+    }
+  })
+})

test/e2e/globalSetup.test.ts

@@ -0,0 +1,21 @@
+/// <reference types="jest-playwright-preset" />
+import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
+
+// This test is to make sure the globalSetup works as expected
+// meaning globalSetup ran and stored the storageState in STORAGE
+describe("globalSetup", () => {
+  beforeEach(async () => {
+    // Create a new context with the saved storage state
+    // so we don't have to logged in
+    const storageState = JSON.parse(STORAGE) || {}
+    await jestPlaywright.resetContext({
+      storageState,
+    })
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
+  })
+
+  it("should keep us logged in using the storageState", async () => {
+    // Make sure the editor actually loaded
+    expect(await page.isVisible("div.monaco-workbench"))
+  })
+})

test/e2e/login.test.ts

@@ -1,38 +1,19 @@
-import { chromium, Page, Browser, BrowserContext } from "playwright"
+/// <reference types="jest-playwright-preset" />
 import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
 
 describe("login", () => {
-  let browser: Browser
-  let page: Page
-  let context: BrowserContext
-
-  beforeAll(async () => {
-    browser = await chromium.launch()
-    context = await browser.newContext()
-  })
-
-  afterAll(async () => {
-    await browser.close()
-  })
-
   beforeEach(async () => {
-    page = await context.newPage()
-  })
-
-  afterEach(async () => {
-    await page.close()
-    // Remove password from local storage
-    await context.clearCookies()
+    await jestPlaywright.resetBrowser()
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
   })
 
   it("should be able to login", async () => {
-    await page.goto(CODE_SERVER_ADDRESS)
     // Type in password
     await page.fill(".password", PASSWORD)
     // Click the submit button and login
     await page.click(".submit")
-    // See the editor
-    const codeServerEditor = await page.isVisible(".monaco-workbench")
-    expect(codeServerEditor).toBeTruthy()
+    await page.waitForLoadState("networkidle")
+    // Make sure the editor actually loaded
+    expect(await page.isVisible("div.monaco-workbench"))
   })
 })

test/e2e/loginPage.test.ts

@@ -0,0 +1,20 @@
+/// <reference types="jest-playwright-preset" />
+
+import { CODE_SERVER_ADDRESS } from "../utils/constants"
+
+describe("login page", () => {
+  beforeEach(async () => {
+    await jestPlaywright.resetContext({
+      logger: {
+        isEnabled: (name, severity) => name === "browser",
+        log: (name, severity, message, args) => console.log(`${name} ${message}`),
+      },
+    })
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
+  })
+
+  it("should see the login page", async () => {
+    // It should send us to the login page
+    expect(await page.title()).toBe("code-server login")
+  })
+})

test/e2e/logout.test.ts

@@ -1,41 +1,20 @@
-import { chromium, Page, Browser, BrowserContext } from "playwright"
-import { CODE_SERVER_ADDRESS, PASSWORD, E2E_VIDEO_DIR } from "../utils/constants"
+/// <reference types="jest-playwright-preset" />
+import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
 
 describe("logout", () => {
-  let browser: Browser
-  let page: Page
-  let context: BrowserContext
-
-  beforeAll(async () => {
-    browser = await chromium.launch()
-    context = await browser.newContext({
-      recordVideo: { dir: E2E_VIDEO_DIR },
-    })
-  })
-
-  afterAll(async () => {
-    await browser.close()
-  })
-
   beforeEach(async () => {
-    page = await context.newPage()
-  })
-
-  afterEach(async () => {
-    await page.close()
-    // Remove password from local storage
-    await context.clearCookies()
+    await jestPlaywright.resetBrowser()
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
   })
 
   it("should be able login and logout", async () => {
-    await page.goto(CODE_SERVER_ADDRESS)
     // Type in password
     await page.fill(".password", PASSWORD)
     // Click the submit button and login
     await page.click(".submit")
-    // See the editor
-    const codeServerEditor = await page.isVisible(".monaco-workbench")
-    expect(codeServerEditor).toBeTruthy()
+    await page.waitForLoadState("networkidle")
+    // Make sure the editor actually loaded
+    expect(await page.isVisible("div.monaco-workbench"))
 
     // Click the Application menu
     await page.click("[aria-label='Application Menu']")
@@ -45,10 +24,16 @@ describe("logout", () => {
     expect(await page.isVisible(logoutButton))
 
     await page.hover(logoutButton)
-
-    await page.click(logoutButton)
-    // it takes a couple seconds to navigate
+    // TODO(@jsjoeio)
+    // Look into how we're attaching the handlers for the logout feature
+    // We need to see how it's done upstream and add logging to the
+    // handlers themselves.
+    // They may be attached too slowly, hence why we need this timeout
     await page.waitForTimeout(2000)
+
+    // Recommended by Playwright for async navigation
+    // https://github.com/microsoft/playwright/issues/1987#issuecomment-620182151
+    await Promise.all([page.waitForNavigation(), page.click(logoutButton)])
     const currentUrl = page.url()
     expect(currentUrl).toBe(`${CODE_SERVER_ADDRESS}/login`)
   })

test/e2e/openHelpAbout.test.ts

@@ -1,71 +1,18 @@
-import { chromium, Page, Browser, BrowserContext, Cookie } from "playwright"
-import { hash } from "../../src/node/util"
-import { CODE_SERVER_ADDRESS, PASSWORD, STORAGE, E2E_VIDEO_DIR } from "../utils/constants"
-import { createCookieIfDoesntExist } from "../utils/helpers"
+/// <reference types="jest-playwright-preset" />
+import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
 
 describe("Open Help > About", () => {
-  let browser: Browser
-  let page: Page
-  let context: BrowserContext
-
-  beforeAll(async () => {
-    browser = await chromium.launch()
+  beforeEach(async () => {
     // Create a new context with the saved storage state
+    // so we don't have to logged in
     const storageState = JSON.parse(STORAGE) || {}
-
-    const cookieToStore = {
-      sameSite: "Lax" as const,
-      name: "key",
-      value: hash(PASSWORD),
-      domain: "localhost",
-      path: "/",
-      expires: -1,
-      httpOnly: false,
-      secure: false,
-    }
-
-    // For some odd reason, the login method used in globalSetup.ts doesn't always work
-    // I don't know if it's on playwright clearing our cookies by accident
-    // or if it's our cookies disappearing.
-    // This means we need an additional check to make sure we're logged in.
-    // We do this by manually adding the cookie to the browser environment
-    // if it's not there at the time the test starts
-    const cookies: Cookie[] = storageState.cookies || []
-    // If the cookie exists in cookies then
-    // this will return the cookies with no changes
-    // otherwise if it doesn't exist, it will create it
-    // hence the name maybeUpdatedCookies
-    //
-    // TODO(@jsjoeio)
-    // The playwright storage thing sometimes works and sometimes doesn't. We should investigate this further
-    // at some point.
-    // See discussion: https://github.com/cdr/code-server/pull/2648#discussion_r575434946
-
-    const maybeUpdatedCookies = createCookieIfDoesntExist(cookies, cookieToStore)
-
-    context = await browser.newContext({
-      storageState: { cookies: maybeUpdatedCookies },
-      recordVideo: { dir: E2E_VIDEO_DIR },
+    await jestPlaywright.resetContext({
+      storageState,
     })
-  })
-
-  afterAll(async () => {
-    // Remove password from local storage
-    await context.clearCookies()
-
-    await context.close()
-    await browser.close()
-  })
-
-  beforeEach(async () => {
-    page = await context.newPage()
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
   })
 
   it("should see a 'Help' then 'About' button in the Application Menu that opens a dialog", async () => {
-    // waitUntil: "domcontentloaded"
-    // In case the page takes a long time to load
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "domcontentloaded" })
-
     // Make sure the editor actually loaded
     expect(await page.isVisible("div.monaco-workbench"))
 

test/jest.e2e.config.ts

@@ -2,11 +2,28 @@
 import type { Config } from "@jest/types"
 
 const config: Config.InitialOptions = {
+  preset: "jest-playwright-preset",
   transform: {
     "^.+\\.ts$": "<rootDir>/node_modules/ts-jest",
   },
   globalSetup: "<rootDir>/utils/globalSetup.ts",
-  testEnvironment: "node",
+  testEnvironmentOptions: {
+    "jest-playwright": {
+      // TODO(@jsjoeio) enable on webkit and firefox
+      // waiting on next playwright release
+      // - https://github.com/microsoft/playwright/issues/6009#event-4536210890
+      // - https://github.com/microsoft/playwright/issues/6020
+      browsers: ["chromium"],
+      // If there's a page error, we don't exit
+      // i.e. something logged in the console
+      exitOnPageError: false,
+      contextOptions: {
+        recordVideo: {
+          dir: "./test/e2e/videos",
+        },
+      },
+    },
+  },
   testPathIgnorePatterns: ["/node_modules/", "/lib/", "/out/", "test/unit"],
   testTimeout: 30000,
   modulePathIgnorePatterns: [

test/package.json

@@ -7,9 +7,13 @@
     "@types/node-fetch": "^2.5.8",
     "@types/supertest": "^2.0.10",
     "jest": "^26.6.3",
+    "jest-playwright-preset": "^1.5.1",
     "jsdom": "^16.4.0",
     "node-fetch": "^2.6.1",
     "playwright": "^1.8.0",
+    "playwright-chromium": "^1.10.0",
+    "playwright-firefox": "^1.10.0",
+    "playwright-webkit": "^1.10.0",
     "supertest": "^6.1.1",
     "ts-jest": "^26.4.4"
   }