Skip to content

Commit

Permalink
fix(tests): fix multiple browsers tests (#1718)
Browse files Browse the repository at this point in the history 
- Use a server only once, not once per browser.
- Avoid require cache for test files.
- Improve testrunner hooks debugging.
  • Loading branch information
@dgozman
dgozman authored Apr 8, 2020
1 parent 6723254 commit 2d57fff
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 54 deletions.
95 changes: 50 additions & 45 deletions test/playwright.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ module.exports.addPlaywrightTests = ({testRunner, products}) => {
delete global.playwright;
});

testRunner.collector().useEnvironment(serverEnvironment); // Custom global environment.
testRunner.collector().useEnvironment(playwrightEnvironment);

for (const product of products) {
const browserTypeEnvironment = new Environment('BrowserType');
browserTypeEnvironment.beforeAll(async state => {
Expand Down Expand Up @@ -145,8 +148,10 @@ module.exports.addPlaywrightTests = ({testRunner, products}) => {
state.page = null;
});

testRunner.collector().useEnvironment(serverEnvironment); // Custom global environment.
testRunner.collector().useEnvironment(playwrightEnvironment);
function loadTest(path) {
require(path);
delete require.cache[require.resolve(path)];
}

describe(product, () => {
// In addition to state, expose these two on global so that describes can access them.
Expand All @@ -164,66 +169,66 @@ module.exports.addPlaywrightTests = ({testRunner, products}) => {

// Page-level tests that are given a browser, a context and a page.
// Each test is launched in a new browser context.
require('./accessibility.spec.js');
require('./autowaiting.spec.js');
require('./click.spec.js');
require('./cookies.spec.js');
require('./dialog.spec.js');
require('./download.spec.js');
require('./elementhandle.spec.js');
require('./emulation.spec.js');
require('./evaluation.spec.js');
require('./frame.spec.js');
require('./focus.spec.js');
require('./input.spec.js');
require('./jshandle.spec.js');
require('./keyboard.spec.js');
require('./mouse.spec.js');
require('./navigation.spec.js');
require('./network.spec.js');
require('./page.spec.js');
require('./queryselector.spec.js');
require('./screenshot.spec.js');
require('./waittask.spec.js');
require('./interception.spec.js');
require('./geolocation.spec.js');
require('./workers.spec.js');
require('./capabilities.spec.js');
require('./permissions.spec.js');
loadTest('./accessibility.spec.js');
loadTest('./autowaiting.spec.js');
loadTest('./click.spec.js');
loadTest('./cookies.spec.js');
loadTest('./dialog.spec.js');
loadTest('./download.spec.js');
loadTest('./elementhandle.spec.js');
loadTest('./emulation.spec.js');
loadTest('./evaluation.spec.js');
loadTest('./frame.spec.js');
loadTest('./focus.spec.js');
loadTest('./input.spec.js');
loadTest('./jshandle.spec.js');
loadTest('./keyboard.spec.js');
loadTest('./mouse.spec.js');
loadTest('./navigation.spec.js');
loadTest('./network.spec.js');
loadTest('./page.spec.js');
loadTest('./queryselector.spec.js');
loadTest('./screenshot.spec.js');
loadTest('./waittask.spec.js');
loadTest('./interception.spec.js');
loadTest('./geolocation.spec.js');
loadTest('./workers.spec.js');
loadTest('./capabilities.spec.js');
loadTest('./permissions.spec.js');

describe.skip(product !== 'Chromium')('[Chromium]', () => {
require('./chromium/chromium.spec.js');
require('./chromium/coverage.spec.js');
require('./chromium/pdf.spec.js');
require('./chromium/session.spec.js');
loadTest('./chromium/chromium.spec.js');
loadTest('./chromium/coverage.spec.js');
loadTest('./chromium/pdf.spec.js');
loadTest('./chromium/session.spec.js');
});
});

// Browser-level tests that are given a browser.
describe('[Driver]', () => {
require('./browser.spec.js');
require('./browsercontext.spec.js');
require('./ignorehttpserrors.spec.js');
require('./popup.spec.js');
loadTest('./browser.spec.js');
loadTest('./browsercontext.spec.js');
loadTest('./ignorehttpserrors.spec.js');
loadTest('./popup.spec.js');
});
});

// Top-level tests that launch Browser themselves.
describe('[Driver]', () => {
require('./defaultbrowsercontext.spec.js');
require('./fixtures.spec.js');
require('./launcher.spec.js');
require('./headful.spec.js');
require('./multiclient.spec.js');
loadTest('./defaultbrowsercontext.spec.js');
loadTest('./fixtures.spec.js');
loadTest('./launcher.spec.js');
loadTest('./headful.spec.js');
loadTest('./multiclient.spec.js');
});

describe.skip(product !== 'Chromium')('[Chromium]', () => {
require('./chromium/launcher.spec.js');
require('./chromium/oopif.spec.js');
require('./chromium/tracing.spec.js');
loadTest('./chromium/launcher.spec.js');
loadTest('./chromium/oopif.spec.js');
loadTest('./chromium/tracing.spec.js');
});

require('./apicoverage.spec.js');
loadTest('./apicoverage.spec.js');

delete global.browserType;
delete global.playwright;
Expand Down
18 changes: 9 additions & 9 deletions utils/testrunner/TestRunner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,7 @@ class TestWorker {
}

async _runHook(testRun, hook, fullName, passTestRun = false) {
await this._willStartHook(hook, fullName);
await this._willStartHook(testRun, hook, fullName);
const timeout = this._testRunner._hookTimeout;
const { promise, terminate } = runUserCallback(hook.body, timeout, passTestRun ? [this._state, testRun] : [this._state]);
this._runningHookTerminate = terminate;
Expand All @@ -289,13 +289,13 @@ class TestWorker {
await this._testRunner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error);
message = `${hook.location.toDetailedString()} - FAILED while running "${hook.name}" in suite "${fullName}": `;
}
await this._didFailHook(hook, fullName, message, error);
await this._didFailHook(testRun, hook, fullName, message, error);
if (testRun)
testRun._error = error;
return false;
}

await this._didCompleteHook(hook, fullName);
await this._didCompleteHook(testRun, hook, fullName);
return true;
}

Expand All @@ -319,19 +319,19 @@ class TestWorker {
debug('testrunner:test')(`[${this._workerId}] ${testRun._result.toUpperCase()} "${testRun.test().fullName()}" (${testRun.test().location()})`);
}

async _willStartHook(hook, fullName) {
debug('testrunner:hook')(`[${this._workerId}] "${hook.name}" started for "${fullName}" (${hook.location})`);
async _willStartHook(testRun, hook, fullName) {
debug('testrunner:hook')(`[${this._workerId}] "${fullName}.${hook.name}" started for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`);
}

async _didFailHook(hook, fullName, message, error) {
debug('testrunner:hook')(`[${this._workerId}] "${hook.name}" FAILED for "${fullName}" (${hook.location})`);
async _didFailHook(testRun, hook, fullName, message, error) {
debug('testrunner:hook')(`[${this._workerId}] "${fullName}.${hook.name}" FAILED for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`);
if (message)
this._testRunner._result.addError(message, error, this);
this._testRunner._result.setResult(TestResult.Crashed, message);
}

async _didCompleteHook(hook, fullName) {
debug('testrunner:hook')(`[${this._workerId}] "${hook.name}" OK for "${fullName}" (${hook.location})`);
async _didCompleteHook(testRun, hook, fullName) {
debug('testrunner:hook')(`[${this._workerId}] "${fullName}.${hook.name}" OK for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`);
}

async shutdown() {
Expand Down

0 comments on commit 2d57fff

Please sign in to comment.