feat: fix and migrate smoke tests to Playwright (microsoft#1961)

* feat: migrate e2e tests to Playwright * fix: OS dependency * fix: race condition * fix: increased launch timeout
matterscan · May 11, 2020 · 1dd0635 · 1dd0635
1 parent f6505d2
commit 1dd0635
Show file tree

Hide file tree

Showing 6 changed files with 192 additions and 150 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -2,6 +2,9 @@ name: Build
 
 on:
   push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
   schedule:
     - cron:  '0 8 * * *'
 
@@ -12,6 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
+      - uses: microsoft/playwright-github-action@v1
       - name: Set up Node.js
         uses: actions/setup-node@v1
         with:
@@ -40,10 +44,11 @@ jobs:
         run: |
           npm run bundle
       - name: Build Tests
-        run: |
-          npm run build-test
+        run: npm run build-test
       - name: Run Smoke Test
         run: |
           npm run ciserver &
           sleep 10
-          npm run test
+          BROWSER=chromium npm run test
+          BROWSER=firefox npm run test
+          BROWSER=webkit npm run test
diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,5 @@ dist/*.js
 dist/fonts/*
 out-ci/
 .DS_Store
-vscode
+vscode
+/typings-test
diff --git a/ci/core.test.ts b/ci/core.test.ts
@@ -1,35 +1,48 @@
-import * as puppeteer from 'puppeteer';
+import * as playwright from 'playwright';
 import { assert } from 'chai';
 
 const APP = 'http://127.0.0.1:8080/dist/core.html';
 
-let browser: puppeteer.Browser;
-let page: puppeteer.Page;
-const width = 800;
-const height = 600;
+let browser: playwright.Browser;
+let page: playwright.Page;
 
-describe('Basic loading', function (): void {
-    this.timeout(20000);
+type BrowserType = "chromium" | "firefox" | "webkit"
 
-    after(() => {
-        browser.close();
-    });
+const browserType: BrowserType = process.env.BROWSER as BrowserType || "chromium"
 
-    it('should fail because page has an error', async () => {
-        browser = await puppeteer.launch({
-            headless: process.argv.indexOf('--headless') !== -1,
-            args: [`--window-size=${width},${height}`, `--no-sandbox`]
-        });
+before(async () => {
+    console.log(`Starting browser: ${browserType}`)
+    browser = await playwright[browserType].launch({
+        headless: process.argv.includes('--headless'),
+    });
+});
+after(async () => {
+    await browser.close();
+});
+beforeEach(async function () {
+    this.timeout(5 * 1000)
+    page = await browser.newPage({
+        viewport: {
+            width: 800,
+            height: 600
+        }
+    });
+});
+afterEach(async () => {
+    await page.close();
+});
 
-        page = (await browser.pages())[0];
+describe('Basic loading', function (): void {
+    this.timeout(20000);
 
+    it('should fail because page has an error', async () => {
         const pageErrors: any[] = [];
         page.on('pageerror', (e) => {
             console.log(e);
             pageErrors.push(e);
         });
 
-        page.on('error', (e) => {
+        page.on('pageerror', (e) => {
             console.log(e);
             pageErrors.push(e);
         });
@@ -46,19 +59,6 @@ describe('Basic loading', function (): void {
 describe('API Integration Tests', function (): void {
     this.timeout(20000);
 
-    before(async function (): Promise<any> {
-        browser = await puppeteer.launch({
-            headless: process.argv.indexOf('--headless') !== -1,
-            args: [`--window-size=${width},${height}`, `--no-sandbox`]
-        });
-        page = (await browser.pages())[0];
-        await page.setViewport({ width, height });
-    });
-
-    after(() => {
-        browser.close();
-    });
-
     beforeEach(async () => {
         await page.goto(APP);
     });
@@ -88,7 +88,7 @@ describe('API Integration Tests', function (): void {
             instance.trigger('keyboard', 'type', {
                 text: 'a'
             });
-            instance.trigger('keyboard', 'undo');
+            instance.getModel().undo();
         })()
         `);
         assert.equal(await page.evaluate(`instance.getModel().getLineContent(1)`), 'from banana import *');
@@ -110,7 +110,7 @@ describe('API Integration Tests', function (): void {
         })()
         `);
 
-        await page.waitFor(1000);
+        await page.waitForTimeout(1000);
 
         assert.deepEqual(await page.evaluate(`
             [
@@ -123,13 +123,13 @@ describe('API Integration Tests', function (): void {
                 instance.getModel().getLineContent(7),
             ]
         `), [
-                '# from banana import *',
-                '# ',
-                '# class Monkey:',
-                '# 	# Bananas the monkey can eat.',
-                '# 	capacity = 10',
-                '# 	def eat(self, N):',
-                '\t\t\'\'\'Make the monkey eat N bananas!\'\'\''
+            '# from banana import *',
+            '# ',
+            '# class Monkey:',
+            '# 	# Bananas the monkey can eat.',
+            '# 	capacity = 10',
+            '# 	def eat(self, N):',
+            '\t\t\'\'\'Make the monkey eat N bananas!\'\'\''
         ]);
     });
 });
diff --git a/ci/tsconfig.json b/ci/tsconfig.json
@@ -7,7 +7,8 @@
         "rootDir": ".",
         "outDir": "../out-ci/",
         "types": [
-            "../node_modules/@types/mocha"
+            "../node_modules/@types/mocha",
+            "../node_modules/@types/node"
         ],
         "sourceMap": true,
         "removeComments": true,