From 7a68420aa13a5d846f55336896382243bd5d21a0 Mon Sep 17 00:00:00 2001 From: Philipp Hancke Date: Wed, 19 Oct 2022 11:03:38 +0200 Subject: [PATCH] add Chrome/Firefox interop tests (#1576) * add interop tests adds interoperability tests between Chrome unstable and Firefox Nightly. These tests are designed to run nightly as github actions. They do not rely on a signaling server, instead use the mocha-based test that is controlling the individual selenium webdriver instances to act as a signaling channel which exchanges offers and answers with candidates. This pattern is described in the testbed repository https://github.com/fippo/testbed from 2016, this is a "more modern" take on the same subject. To run the tests locally, * npm install --no-save chromedriver geckodriver * npm run mocha test/interop/connection --- .github/workflows/interop-tests.yml | 22 +++++ package.json | 2 +- test/interop/connection.js | 57 ++++++++++++ test/steps.js | 44 +++++++++ test/webdriver.js | 109 ++++++++++++++-------- test/webrtcclient.js | 138 ++++++++++++++++++++++++++++ 6 files changed, 333 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/interop-tests.yml create mode 100644 test/interop/connection.js create mode 100644 test/steps.js create mode 100644 test/webrtcclient.js diff --git a/.github/workflows/interop-tests.yml b/.github/workflows/interop-tests.yml new file mode 100644 index 000000000..20829a9f9 --- /dev/null +++ b/.github/workflows/interop-tests.yml @@ -0,0 +1,22 @@ +on: + schedule: + - cron: "30 5 * * *" + push: +jobs: + interop: + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + browserA: [chrome, firefox] + browserB: [firefox, chrome] + bver: ['unstable'] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v3 + - run: npm install + - run: BROWSER=${{matrix.browserA}} BVER=${{matrix.bver}} ./node_modules/travis-multirunner/setup.sh + - run: BROWSER=${{matrix.browserB}} BVER=${{matrix.bver}} ./node_modules/travis-multirunner/setup.sh + - run: Xvfb :99 & + - run: BROWSER_A=${{matrix.browserA}} BROWSER_B=${{matrix.browserB}} BVER=${{matrix.bver}} DISPLAY=:99.0 npm run mocha test/interop/connection.js diff --git a/package.json b/package.json index 5c604b416..e55d03645 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "scripts": { "start": "http-server . -c-1", "test": "npm run eslint && npm run stylelint", - "eslint": "eslint 'src/content/**/*.js'", + "eslint": "eslint 'test/**.js' 'src/content/**/*.js'", "mocha": "mocha --timeout 5000 'src/content/**/test.js'", "stylelint": "stylelint 'src/**/*.css'" }, diff --git a/test/interop/connection.js b/test/interop/connection.js new file mode 100644 index 000000000..9b8d0231a --- /dev/null +++ b/test/interop/connection.js @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. + */ +const {buildDriver} = require('../webdriver'); +const {PeerConnection, MediaDevices} = require('../webrtcclient'); +const steps = require('../steps'); + +const browserA = process.env.BROWSER_A || 'chrome'; +const browserB = process.env.BROWSER_B || 'chrome'; + +describe(`basic interop test ${browserA} => ${browserB}`, function() { + this.retries(3); // retry up to three times. + let drivers; + let clients; + before(async () => { + const options = { + version: process.env.BVER || 'stable', + browserLogging: true, + } + drivers = [ + buildDriver(browserA, options), + buildDriver(browserB, options), + ]; + clients = drivers.map(driver => { + return { + connection: new PeerConnection(driver), + mediaDevices: new MediaDevices(driver), + }; + }); + }); + after(async () => { + await drivers.map(driver => driver.close()); + }); + + it('establishes a connection', async () => { + await Promise.all(drivers); // timeouts in before(Each)? + await steps.step(drivers, (d) => d.get('https://webrtc.github.io/samples/emptypage.html'), 'Empty page loaded'); + await steps.step(clients, (client) => client.connection.create(), 'Created RTCPeerConnection'); + await steps.step(clients, async (client) => { + const stream = await client.mediaDevices.getUserMedia({audio: true, video: true}); + return Promise.all(stream.getTracks().map(async track => { + return client.connection.addTrack(track, stream); + })); + }, 'Acquired and added audio/video stream'); + const offerWithCandidates = await clients[0].connection.setLocalDescription(); + await clients[1].connection.setRemoteDescription(offerWithCandidates); + const answerWithCandidates = await clients[1].connection.setLocalDescription(); + await clients[0].connection.setRemoteDescription(answerWithCandidates); + + await steps.step(drivers, (d) => steps.waitNVideosExist(d, 1), 'Video elements exist'); + await steps.step(drivers, steps.waitAllVideosHaveEnoughData, 'Video elements have enough data'); + }).timeout(30000); +}).timeout(90000);; diff --git a/test/steps.js b/test/steps.js new file mode 100644 index 000000000..e31e1bd41 --- /dev/null +++ b/test/steps.js @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. + */ +const TIMEOUT = 10000; + +function step(drivers, cb, logMessage) { + return Promise.all(drivers.map(driver => { + return cb(driver); + })).then(() => { + if (logMessage) { + console.log(logMessage); + } + }); +} +function waitNVideosExist(driver, n) { + return driver.wait(() => { + return driver.executeScript(n => document.querySelectorAll('video').length === n, n); + }, TIMEOUT); +} + +function waitAllVideosHaveEnoughData(driver) { + return driver.wait(() => { + return driver.executeScript(() => { + const videos = document.querySelectorAll('video'); + let ready = 0; + for (let i = 0; i < videos.length; i++) { + if (videos[i].readyState >= videos[i].HAVE_ENOUGH_DATA) { + ready++; + } + } + return ready === videos.length; + }); + }, TIMEOUT); +} + +module.exports = { + step, + waitNVideosExist, + waitAllVideosHaveEnoughData, +}; diff --git a/test/webdriver.js b/test/webdriver.js index a38c59edd..0c080f6c3 100644 --- a/test/webdriver.js +++ b/test/webdriver.js @@ -1,68 +1,101 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. + */ const os = require('os'); -const fs = require('fs'); const webdriver = require('selenium-webdriver'); const chrome = require('selenium-webdriver/chrome'); const firefox = require('selenium-webdriver/firefox'); const safari = require('selenium-webdriver/safari'); -// setup path for webdriver binaries if (os.platform() === 'win32') { - process.env.PATH += ';C:\\Program Files (x86)\\Microsoft Web Driver\\'; - // FIXME: not sure why node_modules\.bin\ is not enough - process.env.PATH += ';' + process.cwd() + - '\\node_modules\\chromedriver\\lib\\chromedriver\\'; - process.env.PATH += ';' + process.cwd() + - '\\node_modules\\geckodriver'; + process.env.PATH += ';' + process.cwd() + '\\node_modules\\chromedriver\\lib\\chromedriver\\'; + process.env.PATH += ';' + process.cwd() + '\\node_modules\\geckodriver'; } else { process.env.PATH += ':node_modules/.bin'; } function buildDriver(browser = process.env.BROWSER || 'chrome', options = {bver: process.env.BVER}) { - // Firefox options. - let firefoxPath; - if (options.firefoxpath) { - firefoxPath = options.firefoxpath; - } else if (os.platform() == 'linux' && options.bver) { - firefoxPath = 'browsers/bin/firefox-' + options.bver; - } else { - firefoxPath = firefox.Channel.RELEASE; - } - - const firefoxOptions = new firefox.Options() - .setPreference('media.navigator.streams.fake', true) - .setPreference('media.navigator.permission.disabled', true) - .setPreference('xpinstall.signatures.required', false) - .setPreference('media.peerconnection.dtls.version.min', 771) - .setBinary(firefoxPath); - // Chrome options. - let chromeOptions = new chrome.Options() - .addArguments('allow-file-access-from-files') + const chromeOptions = new chrome.Options() + .addArguments('allow-insecure-localhost') .addArguments('use-fake-device-for-media-stream') - .addArguments('use-fake-ui-for-media-stream') - .addArguments('disable-translate') - .addArguments('no-process-singleton-dialog') - .addArguments('mute-audio'); - // ensure chrome.runtime is visible. - chromeOptions.excludeSwitches('test-type'); + .addArguments('allow-file-access-from-files'); + if (options.chromeFlags) { + options.chromeFlags.forEach((flag) => chromeOptions.addArguments(flag)); + } if (options.chromepath) { chromeOptions.setChromeBinaryPath(options.chromepath); - } else if (os.platform() === 'linux' && options.bver) { - chromeOptions.setChromeBinaryPath('browsers/bin/chrome-' + options.bver); + } else if (os.platform() === 'linux' && options.version) { + chromeOptions.setChromeBinaryPath('browsers/bin/chrome-' + options.version); + } + + if (!options.devices || options.headless) { + // GUM doesn't work in headless mode so we need this. See + // https://bugs.chromium.org/p/chromium/issues/detail?id=776649 + chromeOptions.addArguments('use-fake-ui-for-media-stream'); + } else { + // see https://bugs.chromium.org/p/chromium/issues/detail?id=459532#c22 + const domain = 'https://' + (options.devices.domain || 'localhost') + ':' + (options.devices.port || 443) + ',*'; + const exceptions = { + media_stream_mic: {}, + media_stream_camera: {}, + }; + + exceptions.media_stream_mic[domain] = { + last_used: Date.now(), + setting: options.devices.audio ? 1 : 2 // 0: ask, 1: allow, 2: denied + }; + exceptions.media_stream_camera[domain] = { + last_used: Date.now(), + setting: options.devices.video ? 1 : 2 + }; + + chromeOptions.setUserPreferences({ + profile: { + content_settings: { + exceptions: exceptions + } + } + }); } const safariOptions = new safari.Options(); safariOptions.setTechnologyPreview(options.bver === 'unstable'); + // Firefox options. + const firefoxOptions = new firefox.Options(); + let firefoxPath = firefox.Channel.RELEASE; + if (options.firefoxpath) { + firefoxPath = options.firefoxpath; + } else if (os.platform() == 'linux' && options.version) { + firefoxPath = 'browsers/bin/firefox-' + options.version; + } + if (options.headless) { + firefoxOptions.addArguments('-headless'); + } + firefoxOptions.setBinary(firefoxPath); + firefoxOptions.setPreference('media.navigator.streams.fake', true); + firefoxOptions.setPreference('media.navigator.permission.disabled', true); + const driver = new webdriver.Builder() - .setFirefoxOptions(firefoxOptions) .setChromeOptions(chromeOptions) .setSafariOptions(safariOptions) - .forBrowser(browser); - driver.getCapabilities().set('acceptInsecureCerts', true); + .setFirefoxOptions(firefoxOptions) + .forBrowser(browser) + .setChromeService( + new chrome.ServiceBuilder().addArguments('--disable-build-check') + ); + if (browser === 'firefox') { + driver.getCapabilities().set('marionette', true); + driver.getCapabilities().set('acceptInsecureCerts', true); + } return driver.build(); } diff --git a/test/webrtcclient.js b/test/webrtcclient.js new file mode 100644 index 000000000..e545361e0 --- /dev/null +++ b/test/webrtcclient.js @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. + */ +// Disable no-undef since this file is a mix of code executed +// in JS and the browser. +/* eslint no-undef: 0 */ +class MediaStream { + constructor(tracks = []) { + this.tracks = tracks; + this.id = 0; + } + + getTracks() { + return this.tracks; + } + + getAudioTracks() { + return this.getTracks().filter(t => t.kind === 'audio'); + } + + getVideoTracks() { + return this.getTracks().filter(t => t.kind === 'video'); + } +} + +class MediaDevices { + constructor(driver) { + this.driver = driver; + } + + getUserMedia(constraints) { + return this.driver.executeAsyncScript((constraints) => { + const callback = arguments[arguments.length - 1]; + if (!window.localStreams) { + window.localStreams = {}; + } + + return navigator.mediaDevices.getUserMedia(constraints) + .then((stream) => { + window.localStreams[stream.id] = stream; + callback({id: stream.id, tracks: stream.getTracks().map((t) => { + return {id: t.id, kind: t.kind}; + })}); + }, (e) => callback(e)); + }, constraints || {audio: true, video: true}) + .then((streamObj) => { + const stream = new MediaStream(streamObj.tracks); + stream.id = streamObj.id; + return stream; + }); + } +} + +class PeerConnection { + constructor(driver) { + this.driver = driver; + } + + create(rtcConfiguration) { + return this.driver.executeScript(rtcConfiguration => { + window.pc = new RTCPeerConnection(rtcConfiguration); + }, rtcConfiguration); + } + + addTrack(track, stream) { + return this.driver.executeScript((track, stream) => { + stream = localStreams[stream.id]; + track = stream.getTracks().find(t => t.id === track.id); + pc.addTrack(track, stream); + }, track, stream); + } + + createOffer(offerOptions) { + return this.driver.executeAsyncScript((offerOptions) => { + const callback = arguments[arguments.length - 1]; + + pc.createOffer(offerOptions) + .then(callback, callback); + }, offerOptions); + } + createAnswer() { + return this.driver.executeAsyncScript(() => { + const callback = arguments[arguments.length - 1]; + + pc.createAnswer() + .then(callback, callback); + }); + } + + // resolves with non-trickle description including candidates. + setLocalDescription(desc) { + return this.driver.executeAsyncScript((desc) => { + const callback = arguments[arguments.length - 1]; + + pc.onicecandidate = (event) => { + console.log('candidate', event.candidate); + if (!event.candidate) { + pc.onicecandidate = null; + callback(pc.localDescription); + } + }; + pc.setLocalDescription(desc) + .catch(callback); + }, desc); + } + + // TODO: this implicitly creates video elements, is that deseriable? + setRemoteDescription(desc) { + return this.driver.executeAsyncScript(function(desc) { + const callback = arguments[arguments.length - 1]; + + pc.ontrack = function(event) { + const id = event.streams[0].id; + if (document.getElementById('video-' + id)) { + return; + } + const video = document.createElement('video'); + video.id = 'video-' + id; + video.autoplay = true; + video.srcObject = event.streams[0]; + document.body.appendChild(video); + }; + pc.setRemoteDescription(new RTCSessionDescription(desc)) + .then(callback, callback); + }, desc); + } +} + +module.exports = { + PeerConnection, + MediaDevices, + MediaStream, +}; +