devops: teach buildbots to run sanity check script (microsoft#2064)

This should make sure that build actually runs.

browser_patches/checkout_build_archive_upload.sh

@@ -164,14 +164,19 @@ function generate_and_upload_browser_build {
     return 22
   fi
 
+  echo "-- running sanity test"
+  if ! ./$BROWSER_NAME/run_sanity_check.sh; then
+    return 23
+  fi
+
   echo "-- archiving to $ZIP_PATH"
   if ! ./$BROWSER_NAME/archive.sh $ZIP_PATH "$EXTRA_ARCHIVE_ARGS"; then
-    return 23
+    return 24
   fi
 
   echo "-- uploading"
   if ! ./upload.sh $BUILD_BLOB_PATH $ZIP_PATH; then
-    return 24
+    return 25
   fi
   return 0
 }
@@ -206,8 +211,10 @@ else
   elif (( RESULT_CODE == 22 )); then
     FAILED_STEP="./build.sh"
   elif (( RESULT_CODE == 23 )); then
-    FAILED_STEP="./archive.sh"
+    FAILED_STEP="./run_sanity_check.sh"
   elif (( RESULT_CODE == 24 )); then
+    FAILED_STEP="./archive.sh"
+  elif (( RESULT_CODE == 25 )); then
     FAILED_STEP="./upload.sh"
   else
     FAILED_STEP="<unknown step>"

browser_patches/firefox/run_sanity_check.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+set +x
+
+trap "cd $(pwd -P)" EXIT
+cd "$(dirname $0)"
+
+if [[ "$(uname)" == "Darwin" ]]; then
+  node ./sanity.js
+elif [[ "$(uname)" == "Linux" ]]; then
+  xvfb-run --auto-servernum node ./sanity.js
+elif [[ "$(uname)" == MINGW* ]]; then
+  node ./sanity.js
+else
+  echo "ERROR: cannot check sanity on this platform!" 1>&2
+  exit 1;
+fi

browser_patches/firefox/sanity.js

@@ -0,0 +1,28 @@
+const {firefox} = require('../..');
+const os = require('os');
+const path = require('path');
+
+const executablePath = {
+  'darwin': path.join(__dirname, 'checkout', 'obj-build-playwright', 'dist', 'Nightly.app', 'Contents', 'MacOS', 'firefox'),
+  'linux': path.join(__dirname, 'checkout', 'obj-build-playwright', 'dist', 'bin', 'firefox'),
+  'win32': path.join(__dirname, 'checkout', 'obj-build-playwright', 'dist', 'bin', 'firefox.exe'),
+}[os.platform()];
+
+async function checkSanity(options) {
+  const browser = await firefox.launch({...options, executablePath});
+  const context = await browser.newContext();
+  const page = await context.newPage();
+  const result = await page.evaluate(() => 6 * 7);
+  await browser.close();
+  if (result !== 42)
+    throw new Error(`ERROR: computation failed!`);
+  console.log(`SUCCESS: ran firefox with options = ${JSON.stringify(options)}`);
+}
+
+Promise.all([
+  checkSanity({headless: true}),
+  checkSanity({headless: false}),
+]).catch(e => {
+  console.error(e);
+  process.exitCode = 1;
+});

browser_patches/webkit/run_sanity_check.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+set +x
+
+trap "cd $(pwd -P)" EXIT
+cd "$(dirname $0)"
+
+if [[ "$(uname)" == "Darwin" ]]; then
+  node ./sanity.js
+elif [[ "$(uname)" == "Linux" ]]; then
+  xvfb-run --auto-servernum node ./sanity.js
+elif [[ "$(uname)" == MINGW* ]]; then
+  echo "Sanity check on windows is not supported"
+else
+  echo "ERROR: cannot check sanity on this platform!" 1>&2
+  exit 1;
+fi

browser_patches/webkit/sanity.js

@@ -0,0 +1,33 @@
+const {firefox} = require('../..');
+const os = require('os');
+const path = require('path');
+
+const executablePath = {
+  'darwin': path.join(__dirname, 'pw_run.sh'),
+  'linux': path.join(__dirname, 'pw_run.sh'),
+  'win32': undefined,
+}[os.platform()];
+
+// TODO: verify build on windows.
+if (!executablePath)
+  return;
+
+async function checkSanity(options) {
+  const browser = await firefox.launch({...options, executablePath});
+  const context = await browser.newContext();
+  const page = await context.newPage();
+  const result = await page.evaluate(() => 7 * 8);
+  await browser.close();
+  if (result !== 56)
+    throw new Error(`ERROR: computation failed!`);
+  console.log(`SUCCESS: ran webkit with options = ${JSON.stringify(options)}`);
+}
+
+Promise.all([
+  checkSanity({headless: true}),
+  checkSanity({headless: false}),
+]).catch(e => {
+  console.error(e);
+  process.exitCode = 1;
+});
+