Update tests for BrowserStack (#6810)

Update tests to setup webdriver stuff in `jest-environment` and re-use one browser session instead of spawning one for each webdriver call to prevent creating too many BrowserStack sessions.

.circleci/config.yml

@@ -31,6 +31,16 @@ jobs:
             JEST_JUNIT_CLASSNAME: '{filepath}'
       - store_test_results:
           path: ~/repo/reports
+  test-production:
+    docker:
+      - image: circleci/node:8-browsers
+    working_directory: ~/repo
+    steps:
+      - attach_workspace:
+          at: .
+      - run:
+          name: Production Tests
+          command: '[[ ! -z $BROWSERSTACK_USERNAME ]] && yarn testall test/integration/production/ || echo "Not running for PR"'
   deploy:
     docker:
       - image: circleci/node:8-browsers
@@ -55,6 +65,9 @@ workflows:
       - test:
           requires:
             - build
+      - test-production:
+          requires:
+            - build
       - deploy:
           requires:
             - test

.gitignore

@@ -18,11 +18,12 @@ pids
 coverage
 
 # test output
-test/**/out
+test/**/out*
 .DS_Store
 
 # Editors
 **/.idea
 
 # example output
 examples/**/out
+

jest.config.js

@@ -4,10 +4,10 @@ module.exports = {
   testMatch: ['**/*.test.js'],
   verbose: true,
   bail: true,
-  testEnvironment: 'node',
   rootDir: 'test',
   modulePaths: ['<rootDir>/lib'],
   globalSetup: '<rootDir>/jest-global-setup.js',
   globalTeardown: '<rootDir>/jest-global-teardown.js',
+  testEnvironment: '<rootDir>/jest-environment.js',
   coverageReporters: ['text', 'lcov', 'cobertura']
 }

package.json

@@ -70,6 +70,7 @@
     "babel-core": "7.0.0-bridge.0",
     "babel-eslint": "9.0.0",
     "babel-jest": "23.6.0",
+    "browserstack-local": "1.3.7",
     "cheerio": "0.22.0",
     "chromedriver": "2.46.0",
     "clone": "2.1.1",

test/integration/amphtml/test/index.test.js

@@ -1,5 +1,5 @@
 /* eslint-env jest */
-/* global jasmine */
+/* global jasmine, webdriver */
 import { join } from 'path'
 import { readFileSync, writeFileSync } from 'fs'
 import {
@@ -14,7 +14,6 @@ import {
   launchApp,
   killApp
 } from 'next-test-utils'
-import webdriver from 'next-webdriver'
 import cheerio from 'cheerio'
 import amphtmlValidator from 'amphtml-validator'
 const appDir = join(__dirname, '../')

test/integration/app-aspath/test/index.test.js

@@ -1,6 +1,5 @@
 /* eslint-env jest */
-/* global jasmine */
-import webdriver from 'next-webdriver'
+/* global jasmine, webdriver */
 import { readFileSync, writeFileSync } from 'fs'
 import { join } from 'path'
 import {

test/integration/app-document/test/client.js

@@ -1,6 +1,6 @@
 /* eslint-env jest */
+/* global webdriver */
 
-import webdriver from 'next-webdriver'
 import { readFileSync, writeFileSync } from 'fs'
 import { join } from 'path'
 import { check } from 'next-test-utils'

test/integration/app-document/test/csp.js

@@ -1,20 +1,23 @@
 /* eslint-env jest */
-
-import webdriver from 'next-webdriver'
+/* global webdriver */
 
 export default (context, render) => {
   describe('With CSP enabled', () => {
     it('should load inline script by hash', async () => {
       const browser = await webdriver(context.appPort, '/?withCSP=hash')
-      const errLog = await browser.log('browser')
-      expect(errLog.filter((e) => e.source === 'security')).toEqual([])
+      if (browser.log) {
+        const errLog = await browser.log('browser')
+        expect(errLog.filter((e) => e.source === 'security')).toEqual([])
+      }
       await browser.close()
     })
 
     it('should load inline script by nonce', async () => {
       const browser = await webdriver(context.appPort, '/?withCSP=nonce')
-      const errLog = await browser.log('browser')
-      expect(errLog.filter((e) => e.source === 'security')).toEqual([])
+      if (browser.log) {
+        const errLog = await browser.log('browser')
+        expect(errLog.filter((e) => e.source === 'security')).toEqual([])
+      }
       await browser.close()
     })
   })

test/integration/basic/test/dynamic.js

@@ -1,5 +1,5 @@
 /* eslint-env jest */
-import webdriver from 'next-webdriver'
+/* global webdriver */
 import cheerio from 'cheerio'
 import { waitFor, check } from 'next-test-utils'
 
@@ -45,11 +45,13 @@ export default (context, render) => {
           await check(() => browser.elementByCss('body').text(), /Nested 2/)
           await check(() => browser.elementByCss('body').text(), /Browser hydrated/)
 
-          const logs = await browser.log('browser')
+          if (browser.log) {
+            const logs = await browser.log('browser')
 
-          logs.forEach(logItem => {
-            expect(logItem.message).not.toMatch(/Expected server HTML to contain/)
-          })
+            logs.forEach(logItem => {
+              expect(logItem.message).not.toMatch(/Expected server HTML to contain/)
+            })
+          }
         } finally {
           if (browser) {
             await browser.close()

test/integration/basic/test/error-recovery.js

@@ -1,5 +1,5 @@
 /* eslint-env jest */
-import webdriver from 'next-webdriver'
+/* global webdriver */
 import { join } from 'path'
 import { check, File, waitFor, getReactErrorOverlayContent, getBrowserBodyText } from 'next-test-utils'
 
@@ -40,9 +40,8 @@ export default (context, renderViaHTTP) => {
     it('should have installed the react-overlay-editor editor handler', async () => {
       let browser
       const aboutPage = new File(join(__dirname, '../', 'pages', 'hmr', 'about.js'))
-      aboutPage.replace('</div>', 'div')
-
       try {
+        aboutPage.replace('</div>', 'div')
         browser = await webdriver(context.appPort, '/hmr/about')
 
         // react-error-overlay uses the following inline style if an editorHandler is installed
@@ -76,8 +75,10 @@ export default (context, renderViaHTTP) => {
       const aboutPage = new File(join(__dirname, '../', 'pages', 'hmr', 'about.js'))
       try {
         browser = await webdriver(context.appPort, '/hmr/about')
-        const text = await browser.elementByCss('p').text()
-        expect(text).toBe('This is the about page.')
+        await check(
+          () => getBrowserBodyText(browser),
+          /This is the about page/
+        )
 
         aboutPage.replace('</div>', 'div')
 
@@ -147,9 +148,10 @@ export default (context, renderViaHTTP) => {
       const aboutPage = new File(join(__dirname, '../', 'pages', 'hmr', 'about.js'))
       try {
         browser = await webdriver(context.appPort, '/hmr/about')
-        const text = await browser
-          .elementByCss('p').text()
-        expect(text).toBe('This is the about page.')
+        await check(
+          () => getBrowserBodyText(browser),
+          /This is the about page/
+        )
 
         aboutPage.replace('export', 'aa=20;\nexport')
 
@@ -174,9 +176,10 @@ export default (context, renderViaHTTP) => {
       const aboutPage = new File(join(__dirname, '../', 'pages', 'hmr', 'about.js'))
       try {
         browser = await webdriver(context.appPort, '/hmr/about')
-        const text = await browser.elementByCss('p').text()
-
-        expect(text).toBe('This is the about page.')
+        await check(
+          () => getBrowserBodyText(browser),
+          /This is the about page/
+        )
 
         aboutPage.replace('return', 'throw new Error("an-expected-error");\nreturn')
 
@@ -210,8 +213,10 @@ export default (context, renderViaHTTP) => {
       const aboutPage = new File(join(__dirname, '../', 'pages', 'hmr', 'about.js'))
       try {
         browser = await webdriver(context.appPort, '/hmr/about')
-        const text = await browser.elementByCss('p').text()
-        expect(text).toBe('This is the about page.')
+        await check(
+          () => getBrowserBodyText(browser),
+          /This is the about page/
+        )
 
         aboutPage.replace('export default', 'export default {};\nexport const fn =')
 
@@ -249,8 +254,10 @@ export default (context, renderViaHTTP) => {
       const aboutPage = new File(join(__dirname, '../', 'pages', 'hmr', 'about.js'))
       try {
         browser = await webdriver(context.appPort, '/hmr/about')
-        const text = await browser.elementByCss('p').text()
-        expect(text).toBe('This is the about page.')
+        await check(
+          () => getBrowserBodyText(browser),
+          /This is the about page/
+        )
 
         aboutPage.replace('export default', 'export default () => /search/;\nexport const fn =')
 
@@ -288,8 +295,10 @@ export default (context, renderViaHTTP) => {
       const aboutPage = new File(join(__dirname, '../', 'pages', 'hmr', 'about.js'))
       try {
         browser = await webdriver(context.appPort, '/hmr/about')
-        const text = await browser.elementByCss('p').text()
-        expect(text).toBe('This is the about page.')
+        await check(
+          () => getBrowserBodyText(browser),
+          /This is the about page/
+        )
 
         aboutPage.replace('export default', 'export default undefined;\nexport const fn =')
 

test/integration/basic/test/hmr.js

@@ -1,5 +1,5 @@
 /* eslint-env jest */
-import webdriver from 'next-webdriver'
+/* global webdriver */
 import { readFileSync, writeFileSync, renameSync, existsSync } from 'fs'
 import { join } from 'path'
 import { waitFor, check, getBrowserBodyText } from 'next-test-utils'
@@ -135,15 +135,15 @@ export default (context, renderViaHTTP) => {
           // Change the page
           writeFileSync(pagePath, editedContent, 'utf8')
 
-          // wait for 5 seconds
-          await waitFor(5000)
-
           try {
             // Check whether the this page has reloaded or not.
-            const editedPTag = await browser.elementByCss('.hmr-style-page p')
-            const editedFontSize = await editedPTag.getComputedCss('font-size')
-
-            expect(editedFontSize).toBe('200px')
+            await check(
+              async () => {
+                const editedPTag = await browser.elementByCss('.hmr-style-page p')
+                return editedPTag.getComputedCss('font-size')
+              },
+              /200px/
+            )
           } finally {
             // Finally is used so that we revert the content back to the original regardless of the test outcome
             // restore the about page content.
@@ -173,14 +173,14 @@ export default (context, renderViaHTTP) => {
           // Change the page
           writeFileSync(pagePath, editedContent, 'utf8')
 
-          // wait for 5 seconds
-          await waitFor(5000)
-
           // Check whether the this page has reloaded or not.
-          const editedPTag = await browser.elementByCss('.hmr-style-page p')
-          const editedFontSize = await editedPTag.getComputedCss('font-size')
-
-          expect(editedFontSize).toBe('200px')
+          await check(
+            async () => {
+              const editedPTag = await browser.elementByCss('.hmr-style-page p')
+              return editedPTag.getComputedCss('font-size')
+            },
+            /200px/
+          )
         } finally {
           if (browser) {
             await browser.close()

test/integration/basic/test/process-env.js

@@ -1,5 +1,5 @@
 /* eslint-env jest */
-import webdriver from 'next-webdriver'
+/* global webdriver */
 
 export default (context) => {
   describe('process.env', () => {

test/integration/client-404/test/client-navigation.js

@@ -1,6 +1,5 @@
 /* eslint-env jest */
-
-import webdriver from 'next-webdriver'
+/* global webdriver */
 
 export default (context) => {
   describe('Client Navigation 404', () => {

test/integration/client-navigation/test/index.test.js

@@ -1,7 +1,6 @@
 /* eslint-env jest */
-/* global jasmine */
+/* global jasmine, webdriver */
 import { join } from 'path'
-import webdriver from 'next-webdriver'
 import renderingSuite from './rendering'
 import {
   waitFor,

test/integration/config/test/client.js

@@ -1,6 +1,6 @@
 /* eslint-env jest */
+/* global webdriver */
 
-import webdriver from 'next-webdriver'
 import { waitFor } from 'next-test-utils' /* check, File */
 import { readFileSync, writeFileSync } from 'fs'
 import { join } from 'path'

test/integration/custom-server/test/index.test.js

@@ -1,9 +1,8 @@
 /* eslint-env jest */
-/* global jasmine */
+/* global jasmine, webdriver */
 import { join } from 'path'
 import getPort from 'get-port'
 import clone from 'clone'
-import webdriver from 'next-webdriver'
 import {
   initNextServerScript,
   killApp,

test/integration/export/test/browser.js

@@ -1,5 +1,5 @@
 /* eslint-env jest */
-import webdriver from 'next-webdriver'
+/* global webdriver */
 import { check, getBrowserBodyText } from 'next-test-utils'
 
 export default function (context) {

test/integration/export/test/dev.js

@@ -1,5 +1,5 @@
 /* eslint-env jest */
-import webdriver from 'next-webdriver'
+/* global webdriver */
 import { renderViaHTTP, getBrowserBodyText, check } from 'next-test-utils'
 import cheerio from 'cheerio'
 

test/integration/ondemand/test/index.test.js

@@ -1,8 +1,7 @@
 /* eslint-env jest */
-/* global jasmine */
+/* global jasmine, webdriver */
 import { join, resolve } from 'path'
 import { existsSync } from 'fs'
-import webdriver from 'next-webdriver'
 import AbortController from 'abort-controller'
 import {
   renderViaHTTP,

test/integration/page-extensions/test/hmr.js

@@ -1,5 +1,5 @@
 /* eslint-env jest */
-import webdriver from 'next-webdriver'
+/* global webdriver */
 import { readFileSync, writeFileSync } from 'fs'
 import { join } from 'path'
 import { waitFor } from 'next-test-utils'

test/integration/production-config/test/index.test.js

@@ -1,5 +1,5 @@
 /* eslint-env jest */
-/* global jasmine */
+/* global jasmine, webdriver */
 import { join } from 'path'
 import {
   nextServer,
@@ -8,7 +8,6 @@ import {
   stopApp,
   runNextCommand
 } from 'next-test-utils'
-import webdriver from 'next-webdriver'
 
 jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5
 

test/integration/production/test/dynamic.js

@@ -1,5 +1,5 @@
 /* eslint-env jest */
-import webdriver from 'next-webdriver'
+/* global webdriver */
 import cheerio from 'cheerio'
 import { waitFor, check } from 'next-test-utils'