infrastructure (ci, bundling) work and WPT tests with saucelabs

package.json

@@ -11,8 +11,7 @@
     "deploy": "npm run build && ./deploy.sh && gh-pages build",
     "serve:wpt": "sirv start test/wpt/ --port 8081 --quite",
     "serve:demo": "sirv start demo",
-    "test:sauce:setup": " ./test/setup/start-tunnel.sh",
-    "test:sauce:run": "./test/setup/wpt-repo.sh && TEST_ENV=sauce node test/webdriver/index.js --unhandled-rejections=strict",
+    "test:sauce:run": "./test/setup/wpt-repo.sh && TEST_ENV=sauce node test/webdriver/index.js",
     "test:webdriver:local": "./test/setup/wpt-repo.sh && TEST_ENV=local node test/webdriver/index.js"
   },
   "repository": {

test/setup/wpt-repo.sh

@@ -2,17 +2,15 @@
 
 PROJ_DIR="$(cd "$(dirname "$1")" && pwd)"
 
-
 WPT_DIR="$PROJ_DIR/test/wpt"
 
 # if directory does not exist, clone fresh-stuff from WPT master
 if [ ! -d "$WPT_DIR" ]; then
-  echo "WPT folder not found, cloning"
+  echo "WPT folder not found, cloning from remote..."
   git clone https://github.com/web-platform-tests/wpt $WPT_DIR
 fi
 
 if [ ! -d "$WPT_DIR/polyfills" ]; then
-  echo "creating polyfills folder"
   mkdir "$WPT_DIR/polyfills"
 fi
 

test/tests.config.json

@@ -3,17 +3,35 @@
     "/polyfills/scroll-timeline.js"
   ],
   "harnessTests": [
-    "/scroll-animations/current-time-nan.html"
+    "/scroll-animations/current-time-nan.html",
+    "/scroll-animations/current-time-root-scroller.html",
+    "/scroll-animations/current-time-writing-modes.html",
+    "/scroll-animations/current-time.html",
+    "/scroll-animations/scroll-animation.html",
+    "/scroll-animations/scroll-timeline-invalidation.html",
+    "/scroll-animations/scroll-timeline-phases.tentative.html",
+    "/scroll-animations/setting-current-time.html",
+    "/scroll-animations/setting-playback-rate.html"
   ],
   "browsers": {
     "local": [
       "chrome"
     ],
     "sauce": {
+      "edge": {
+        "browserName": "MicrosoftEdge",
+        "browserVersion": "latest",
+        "platformName": "Windows 10"
+      },
       "chrome": {
         "browserName": "chrome",
         "browserVersion": "latest",
         "platformName": "Windows 10"
+      },
+      "firefox": {
+        "browserName": "firefox",
+        "platformName": "Windows 10",
+        "browserVersion": "latest"
       }
     }
   }

test/webdriver/driver-creator.js

@@ -0,0 +1,49 @@
+const promise = require("selenium-webdriver");
+const webdriver = require("selenium-webdriver");
+const path = require("path");
+const {browsers} = require("../tests.config.json");
+
+const driversBinaries = {
+    // TODO: selenium-webdriver package does not know Edge can run on Mac yet, driver for Mac in the path below is valid.
+    // Error: The WebDriver for Edge could not be found on the current PATH. Please download the latest version of MicrosoftWebDriver.exe
+    "edge": path.join(__dirname, '../local-drivers/mac/msedgedriver'),
+    "chrome": path.join(__dirname, '../local-drivers/mac/chromedriver')
+}
+
+promise.USE_PROMISE_MANAGER = false;
+
+function createSeleniumDrivers ({sauceName, sauceKey, tunnelId, testEnv}) {
+    let drivers = new Map();
+    if (testEnv !== 'sauce') {
+        browsers.local.forEach(browser => {
+            let browserConfig = require("selenium-webdriver/" + browser);
+            browserConfig.setDefaultService(new browserConfig.ServiceBuilder(driversBinaries[browser]).build());
+            drivers.set(browser, new webdriver.Builder().forBrowser(browser).withCapabilities(webdriver.Capabilities[browser]()))
+        })
+        return drivers;
+    }
+    Object.keys(browsers.sauce).forEach(browser => {
+        let currentBrowser = browsers.sauce[browser];
+        drivers.set(browser, new webdriver.Builder()
+            .withCapabilities({
+                browserName: currentBrowser.browserName,
+                platformName: currentBrowser.platformName,
+                browserVersion: currentBrowser.browserVersion,
+                "sauce:options": {
+                    username: sauceName,
+                    accessKey: sauceKey,
+                    build: `WPT Harness Tests for ScrollTimeline ${currentBrowser.browserName} : ${currentBrowser.browserVersion} for ${currentBrowser.platformName}`,
+                    name: "WPT Harness Tests for ScrollTimeline",
+                    maxDuration: 3600,
+                    idleTimeout: 1000,
+                    tags: ["scroll-animations", "polyfill", "scroll-timeline", "wpt-harness-tests", "mocha"],
+                    'tunnelIdentifier': tunnelId
+                },
+            })
+            .usingServer("https://ondemand.saucelabs.com/wd/hub")
+        )
+    });
+    return drivers;
+}
+
+module.exports = createSeleniumDrivers

test/webdriver/index.js

@@ -1,28 +1,30 @@
+// vendor
 require("dotenv").config();
 const webdriver = require("selenium-webdriver");
-const builder = require('./driver-builder');
-
 const sirv = require("sirv");
+const { createServer } = require("http");
+const SauceLabs = require("saucelabs").default;
 
-const {createServer} = require("http");
-
-const {harnessTests} = require("../tests.config.json");
-
-const {magenta, cyan, red, green, black} = require("kleur");
-
-const  SauceLabs = require('saucelabs').default;
-
+// ours
+const createSeleniumDrivers = require('./driver-creator');
+const reporter = require("./reporter");
+const { harnessTests } = require("../tests.config.json");
 
+// Env configs
+//
 // TODO: configure / accept as cli args
 const port = 8081;
 const origin = "localhost"
-
-// firefox requires a protocol even for localhost origins
-const baseUrl = `http://${origin}:${port}`;
+const baseUrl = `http://${origin}:${port}`; // firefox requires a protocol even for localhost origins
 const WPT_DIR = process.env.WPT_DIR || ".";
+const sauceName = process.env.SAUCE_NAME || "";
+const sauceKey = process.env.SAUCE_KEY || "";
+const tunnelId = process.env.SC_TUNNEL_ID || "";
+const testEnv = process.env.TEST_ENV && process.env.TEST_ENV.toLowerCase() === 'sauce' ? 'sauce' : 'local' ;
+
+const drivers = createSeleniumDrivers({sauceName, sauceKey, tunnelId, testEnv});
 
-async function testPage(driver, testPath) {
-    await driver.get(baseUrl + testPath);
+async function collectHarnessTestResults(driver, testPath) {
     let resultsScriptElement = await driver.findElement(
         webdriver.By.id("__testharness__results__")
     );
@@ -38,7 +40,7 @@ async function testPage(driver, testPath) {
     })
 }
 
-async function startLocalServer() {
+async function serveWPT() {
     let server = createServer(sirv(WPT_DIR, {quite: true}));
     return new Promise((resolve, reject) => {
         server.listen(port, origin, err => {
@@ -50,38 +52,31 @@ async function startLocalServer() {
     })
 }
 
-// async function stopLocalServer(instance) {
-//     return new Promise((resolve, reject) => {
-//         instance.close(err => {
-//             if (err) {
-//                 return reject(err);
-//             }
-//             resolve(true)
-//         })
-//     })
-// }
-
 async function runWebDriverTests() {
     let server;
+    let driver;
+    let testResults = new Map();
     //TODO: convert JavaScript's rejection handling from try{}catch{} to:
     // let [resolved, rejected] = await fn()
     try {
-        server = await startLocalServer()
+        server = await serveWPT()
     } catch (e) {
+        console.error("Could not start local dev server..")
         throw new Error(e)
     }
-    let testResults = new Map();
-    for (const [browser, driverBuilder] of builder.drivers.entries()) {
-        let currentBrowserResults = []
-        let driver = await driverBuilder.build()
-        for (let i = 0; i < harnessTests.length; i++) {
+    for (const [browser, driverInstance] of drivers.entries()) {
+        let currentBrowserResults = [];
+        try {
+            driver = await driverInstance.build();
+        } catch (e) {
+            console.error("Error creating driver...")
+            console.error(e);
+        }
+        for (let i=0; i < harnessTests.length; i++) {
             let testPath = harnessTests[i];
-            try {
-                let thisRes = await testPage(driver, harnessTests[i]);
-                currentBrowserResults.push(thisRes)
-            } catch (e) {
-                console.warn('Error happened when testing ' + testPath)
-            }
+            await driver.get(baseUrl + testPath);
+            let res = await collectHarnessTestResults(driver);
+            currentBrowserResults.push(res)
         }
         // cleanup before going to next browser's driver
         await driver.quit();
@@ -94,57 +89,22 @@ async function runWebDriverTests() {
     })
 }
 
-async function reporter(results) {
-    let passes = 0;
-    let failures = 0;
-    let suitesOK = 0;
-    let suitesErr = 0;
-    results.forEach((browserResults, browser) => {
-        Object(browserResults).forEach(suite => {
-            console.log(`${magenta().bold([browser.toUpperCase()])} ${cyan().bold("testing: " + suite.test)}`)
-            if (suite.status === 0) {
-                console.log(green().bold("Harness Test: OK"))
-                suitesOK++
-            } else {
-                console.log(red().bold("Harness Test: Error"))
-                console.log(red(suite.message))
-                suitesErr++
-            }
-            console.log("Details:")
-            suite.tests.forEach(t => {
-                if (t.status === 0) {
-                    passes++;
-                    console.log(`\t${green().bold("PASS")} ${green(t.name)} `)
-                } else {
-                    failures++;
-                    console.log(`\t${red().bold("FAIL")} ${red(t.name)} `)
-                    console.log(`\t     ${ red("Details: ") + t.message }`)
-                }
-            })
-        })
-    })
 
-    console.log(black().bgWhite(` SUMMARY `))
-    console.log(`Harness Tests Successfuly Completed: ${suitesOK}/${suitesOK + suitesErr}`);
-    console.log(`Tests failed: ${failures}`);
-    console.log(`Tests passed: ${passes}\n\n\n`);
+(async () => {
+    let sc, sauceAccount;
 
-    return new Promise((resolve) => {
-        if( failures > 0 ) {
-            resolve(1)
-        } else {
-            resolve(0)
-        }
-    })
-}
+    if( testEnv === "sauce") {
+        sauceAccount = new SauceLabs({ user: sauceName, key: sauceKey });
+        sc = await sauceAccount.startSauceConnect({
+            tunnelIdentifier: tunnelId
+        })
+    }
 
-(async () => {
-    const sauceAccount = new SauceLabs({ user: process.env.SAUCE_NAME, key: process.env.SAUCE_KEY });
-    const sc = await sauceAccount.startSauceConnect({
-        tunnelIdentifier: process.env.SC_TUNNEL_ID
-    })
     const results = await runWebDriverTests();
     const exitCode = await reporter(results)
-    await sc.close();
+    if( testEnv === "sauce" ) {
+        await sc.close();
+    }
     process.exit(exitCode);
 })()
+