Skip to content

Commit

Permalink
add Chrome/Firefox interop tests (webrtc#1576)
Browse files Browse the repository at this point in the history
* add interop tests

adds interoperability tests between Chrome unstable and Firefox
Nightly. These tests are designed to run nightly as github actions.

They do not rely on a signaling server, instead use the mocha-based
test that is controlling the individual selenium webdriver instances
to act as a signaling channel which exchanges offers and answers with
candidates.
This pattern is described in the testbed repository
  https://github.com/fippo/testbed
from 2016, this is a "more modern" take on the same subject.

To run the tests locally,
* npm install --no-save chromedriver geckodriver
* npm run mocha test/interop/connection
  • Loading branch information
fippo authored Oct 19, 2022
1 parent 5990ba4 commit 7a68420
Show file tree
Hide file tree
Showing 6 changed files with 333 additions and 39 deletions.
22 changes: 22 additions & 0 deletions .github/workflows/interop-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
on:
schedule:
- cron: "30 5 * * *"
push:
jobs:
interop:
runs-on: ubuntu-latest
timeout-minutes: 5
strategy:
fail-fast: false
matrix:
browserA: [chrome, firefox]
browserB: [firefox, chrome]
bver: ['unstable']
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v3
- run: npm install
- run: BROWSER=${{matrix.browserA}} BVER=${{matrix.bver}} ./node_modules/travis-multirunner/setup.sh
- run: BROWSER=${{matrix.browserB}} BVER=${{matrix.bver}} ./node_modules/travis-multirunner/setup.sh
- run: Xvfb :99 &
- run: BROWSER_A=${{matrix.browserA}} BROWSER_B=${{matrix.browserB}} BVER=${{matrix.bver}} DISPLAY=:99.0 npm run mocha test/interop/connection.js
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"scripts": {
"start": "http-server . -c-1",
"test": "npm run eslint && npm run stylelint",
"eslint": "eslint 'src/content/**/*.js'",
"eslint": "eslint 'test/**.js' 'src/content/**/*.js'",
"mocha": "mocha --timeout 5000 'src/content/**/test.js'",
"stylelint": "stylelint 'src/**/*.css'"
},
Expand Down
57 changes: 57 additions & 0 deletions test/interop/connection.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree.
*/
const {buildDriver} = require('../webdriver');
const {PeerConnection, MediaDevices} = require('../webrtcclient');
const steps = require('../steps');

const browserA = process.env.BROWSER_A || 'chrome';
const browserB = process.env.BROWSER_B || 'chrome';

describe(`basic interop test ${browserA} => ${browserB}`, function() {
this.retries(3); // retry up to three times.
let drivers;
let clients;
before(async () => {
const options = {
version: process.env.BVER || 'stable',
browserLogging: true,
}
drivers = [
buildDriver(browserA, options),
buildDriver(browserB, options),
];
clients = drivers.map(driver => {
return {
connection: new PeerConnection(driver),
mediaDevices: new MediaDevices(driver),
};
});
});
after(async () => {
await drivers.map(driver => driver.close());
});

it('establishes a connection', async () => {
await Promise.all(drivers); // timeouts in before(Each)?
await steps.step(drivers, (d) => d.get('https://webrtc.github.io/samples/emptypage.html'), 'Empty page loaded');
await steps.step(clients, (client) => client.connection.create(), 'Created RTCPeerConnection');
await steps.step(clients, async (client) => {
const stream = await client.mediaDevices.getUserMedia({audio: true, video: true});
return Promise.all(stream.getTracks().map(async track => {
return client.connection.addTrack(track, stream);
}));
}, 'Acquired and added audio/video stream');
const offerWithCandidates = await clients[0].connection.setLocalDescription();
await clients[1].connection.setRemoteDescription(offerWithCandidates);
const answerWithCandidates = await clients[1].connection.setLocalDescription();
await clients[0].connection.setRemoteDescription(answerWithCandidates);

await steps.step(drivers, (d) => steps.waitNVideosExist(d, 1), 'Video elements exist');
await steps.step(drivers, steps.waitAllVideosHaveEnoughData, 'Video elements have enough data');
}).timeout(30000);
}).timeout(90000);;
44 changes: 44 additions & 0 deletions test/steps.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree.
*/
const TIMEOUT = 10000;

function step(drivers, cb, logMessage) {
return Promise.all(drivers.map(driver => {
return cb(driver);
})).then(() => {
if (logMessage) {
console.log(logMessage);
}
});
}
function waitNVideosExist(driver, n) {
return driver.wait(() => {
return driver.executeScript(n => document.querySelectorAll('video').length === n, n);
}, TIMEOUT);
}

function waitAllVideosHaveEnoughData(driver) {
return driver.wait(() => {
return driver.executeScript(() => {
const videos = document.querySelectorAll('video');
let ready = 0;
for (let i = 0; i < videos.length; i++) {
if (videos[i].readyState >= videos[i].HAVE_ENOUGH_DATA) {
ready++;
}
}
return ready === videos.length;
});
}, TIMEOUT);
}

module.exports = {
step,
waitNVideosExist,
waitAllVideosHaveEnoughData,
};
109 changes: 71 additions & 38 deletions test/webdriver.js
Original file line number Diff line number Diff line change
@@ -1,68 +1,101 @@
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree.
*/
const os = require('os');
const fs = require('fs');

const webdriver = require('selenium-webdriver');
const chrome = require('selenium-webdriver/chrome');
const firefox = require('selenium-webdriver/firefox');
const safari = require('selenium-webdriver/safari');

// setup path for webdriver binaries
if (os.platform() === 'win32') {
process.env.PATH += ';C:\\Program Files (x86)\\Microsoft Web Driver\\';
// FIXME: not sure why node_modules\.bin\ is not enough
process.env.PATH += ';' + process.cwd() +
'\\node_modules\\chromedriver\\lib\\chromedriver\\';
process.env.PATH += ';' + process.cwd() +
'\\node_modules\\geckodriver';
process.env.PATH += ';' + process.cwd() + '\\node_modules\\chromedriver\\lib\\chromedriver\\';
process.env.PATH += ';' + process.cwd() + '\\node_modules\\geckodriver';
} else {
process.env.PATH += ':node_modules/.bin';
}

function buildDriver(browser = process.env.BROWSER || 'chrome', options = {bver: process.env.BVER}) {
// Firefox options.
let firefoxPath;
if (options.firefoxpath) {
firefoxPath = options.firefoxpath;
} else if (os.platform() == 'linux' && options.bver) {
firefoxPath = 'browsers/bin/firefox-' + options.bver;
} else {
firefoxPath = firefox.Channel.RELEASE;
}

const firefoxOptions = new firefox.Options()
.setPreference('media.navigator.streams.fake', true)
.setPreference('media.navigator.permission.disabled', true)
.setPreference('xpinstall.signatures.required', false)
.setPreference('media.peerconnection.dtls.version.min', 771)
.setBinary(firefoxPath);

// Chrome options.
let chromeOptions = new chrome.Options()
.addArguments('allow-file-access-from-files')
const chromeOptions = new chrome.Options()
.addArguments('allow-insecure-localhost')
.addArguments('use-fake-device-for-media-stream')
.addArguments('use-fake-ui-for-media-stream')
.addArguments('disable-translate')
.addArguments('no-process-singleton-dialog')
.addArguments('mute-audio');
// ensure chrome.runtime is visible.
chromeOptions.excludeSwitches('test-type');
.addArguments('allow-file-access-from-files');
if (options.chromeFlags) {
options.chromeFlags.forEach((flag) => chromeOptions.addArguments(flag));
}

if (options.chromepath) {
chromeOptions.setChromeBinaryPath(options.chromepath);
} else if (os.platform() === 'linux' && options.bver) {
chromeOptions.setChromeBinaryPath('browsers/bin/chrome-' + options.bver);
} else if (os.platform() === 'linux' && options.version) {
chromeOptions.setChromeBinaryPath('browsers/bin/chrome-' + options.version);
}

if (!options.devices || options.headless) {
// GUM doesn't work in headless mode so we need this. See
// https://bugs.chromium.org/p/chromium/issues/detail?id=776649
chromeOptions.addArguments('use-fake-ui-for-media-stream');
} else {
// see https://bugs.chromium.org/p/chromium/issues/detail?id=459532#c22
const domain = 'https://' + (options.devices.domain || 'localhost') + ':' + (options.devices.port || 443) + ',*';
const exceptions = {
media_stream_mic: {},
media_stream_camera: {},
};

exceptions.media_stream_mic[domain] = {
last_used: Date.now(),
setting: options.devices.audio ? 1 : 2 // 0: ask, 1: allow, 2: denied
};
exceptions.media_stream_camera[domain] = {
last_used: Date.now(),
setting: options.devices.video ? 1 : 2
};

chromeOptions.setUserPreferences({
profile: {
content_settings: {
exceptions: exceptions
}
}
});
}

const safariOptions = new safari.Options();
safariOptions.setTechnologyPreview(options.bver === 'unstable');

// Firefox options.
const firefoxOptions = new firefox.Options();
let firefoxPath = firefox.Channel.RELEASE;
if (options.firefoxpath) {
firefoxPath = options.firefoxpath;
} else if (os.platform() == 'linux' && options.version) {
firefoxPath = 'browsers/bin/firefox-' + options.version;
}
if (options.headless) {
firefoxOptions.addArguments('-headless');
}
firefoxOptions.setBinary(firefoxPath);
firefoxOptions.setPreference('media.navigator.streams.fake', true);
firefoxOptions.setPreference('media.navigator.permission.disabled', true);

const driver = new webdriver.Builder()
.setFirefoxOptions(firefoxOptions)
.setChromeOptions(chromeOptions)
.setSafariOptions(safariOptions)
.forBrowser(browser);
driver.getCapabilities().set('acceptInsecureCerts', true);
.setFirefoxOptions(firefoxOptions)
.forBrowser(browser)
.setChromeService(
new chrome.ServiceBuilder().addArguments('--disable-build-check')
);

if (browser === 'firefox') {
driver.getCapabilities().set('marionette', true);
driver.getCapabilities().set('acceptInsecureCerts', true);
}
return driver.build();
}

Expand Down
Loading

0 comments on commit 7a68420

Please sign in to comment.