Programmatically start server from compat test scripts

EXTRA: If tests fail return an error code.

Author: brizental

benchmarks/compat/tests/browserstack.test.js

@@ -5,7 +5,10 @@
 import browserstack from "browserstack-local";
 import webdriver from "selenium-webdriver";
 
-import { runWebTest, fastSelenium } from "./utils.js"
+import { runWebTest, fastSelenium, App } from "./utils.js"
+
+const app = new App();
+app.start();
 
 fastSelenium();
 
@@ -76,6 +79,8 @@ for (const browser of BROWSERS) {
   }
 }
 
+app.stop();
+
 // Stop BrowserStack.Local.
 await new Promise(resolve => {
   bs.stop(

benchmarks/compat/tests/local.test.js

@@ -5,7 +5,10 @@
 import webdriver from "selenium-webdriver"
 import { Capabilities } from "selenium-webdriver";
 
-import { runWebTest } from "./utils.js"
+import { App, runWebTest } from "./utils.js"
+
+const app = new App();
+await app.start();
 
 await (async function() {
   let driver;
@@ -17,9 +20,11 @@ await (async function() {
 
     await runWebTest(driver);
   } catch(e) {
-    // We just catch this so that even if the test fails we quit the driver.
-    // Either way there will be a log saying if the test failed.
+    // Make sure the process exits with an error code
+    process.exitCode = 1;
   }
 
   driver.quit();
 }());
+
+await app.stop();

benchmarks/compat/tests/utils.js

@@ -4,13 +4,48 @@
 
 import http from "http";
 import https from "https";
+import path, { dirname } from "path";
+import { fileURLToPath } from "url";
 
 import { By, until } from "selenium-webdriver";
+import webpack from "webpack";
+import DevServer from "webpack-dev-server";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const PORT = 3000;
+
+export class App {
+  constructor() {
+    const compiler = webpack({
+      mode: "development",
+      entry: path.join(__dirname, "../index.js")
+    });
+
+    this.server = new DevServer({
+      static: {
+        directory: path.join(__dirname, "../"),
+      },
+      port: 3000,
+    }, compiler);
+  }
+
+  async start() {
+    console.log("Starting server...");
+    await this.server.start();
+  }
+
+  async stop() {
+    console.log("Stopping server...");
+    await this.server.stop();
+  }
+}
 
 /**
  * Executes a smoke test for @mozilla/glean/web.
  *
- * This assumes a server is running on localhost:8081 with the sample page.
+ * This will execute the sample app in port 3000.
  *
  * @param {*} driver A selenium webdriver.
  */
@@ -19,18 +54,18 @@ export async function runWebTest(driver) {
     // Loading the sample webpage will record a metric and submit a ping.
     // If the ping is submitted succesfully an element in the DOM with the id `msg`
     // will receive the text "Ping submitted succesfully."
-    await driver.get("http://localhost:8081");
+    await driver.get(`http://localasdfhost:${PORT}/`);
     // Give it time to send the ping request.
     const successTextContainer = await driver.findElement(By.id("msg"));
     await driver.wait(
       until.elementTextIs(
         successTextContainer,
         "Ping submitted succesfully."
-      ), 5000);
+      ), 60* 5000);
 
     console.log("Test passed.");
   } catch(e) {
-    console.log("Test failed.");
+    console.log("Test failed.", e);
     // Bubble error up to calling function.
     throw e;
   }

benchmarks/package.json

@@ -13,10 +13,8 @@
     "size:docs:dry": "DRY_RUN=1 node --experimental-json-modules size/docs.js",
     "size:docs": "node --experimental-json-modules size/docs.js",
     "compat:glean": "glean translate compat/metrics.yaml compat/pings.yaml -f javascript -o compat/generated",
-    "compat:build": "npm run compat:glean && webpack ./compat/index.js --output-path=./compat/dist --output-filename=index.js --mode=development",
-    "compat:dev": "npm run compat:build && webpack serve ./compat/index.js --static-directory=./compat --mode=development --open ",
-    "compat:test": "node --experimental-json-modules compat/tests/local.test.js",
-    "compat:test:browserstack": "node --experimental-json-modules compat/tests/browserstack.test.js"
+    "compat:test": "npm run compat:glean && node --experimental-json-modules compat/tests/local.test.js",
+    "compat:test:browserstack": "npm run compat:glean && node --experimental-json-modules compat/tests/browserstack.test.js"
   },
   "author": "The Glean Team <glean-team@mozilla.com>",
   "license": "MPL-2.0",