Bug 1363428 - Add reftest-specific endpoints to Marionette, r=ato

This adds commands to start a reftest session, run a test, and end the
session. It as assumed that after you start a reftest session you will
just run reftests until you end the session. When starting a session
the user provides a string indicating when screenshots should be
taken, and an object mapping urls to a count of the number of times
that url is expected to be used in the session, to help with
caching. Running the tests takes a url to a test, an expected status,
a timeout, and a nested list of possible references, in which each
entry at a specific level is combined by OR and nested references are
combined by AND.

The implementation is heavilly inspired by the existing reftest
harness, starting a minimal window with no tabs, and loading the urls
directly in there. In order to get a screenshot in the e10s case we
have to pass the DRAW_VIEW and USE_WIDGET_LAYERS flags when taking the
screenshot.

For performance we heavily cache canvases; for references that will be
repeated we cache the full canvas with image, and we also cache a
single canvas to use for all other screenshots to avoid the overhead
of repeatedly creating a new canvas element.

MozReview-Commit-ID: JOFvtmH7tg

layout/tools/reftest/reftest-content.js

@@ -447,6 +447,7 @@ function WaitForTestEnd(contentRootElement, inPrintMode, spellCheckedElements) {
     var state = STATE_WAITING_TO_FIRE_INVALIDATE_EVENT;
 
     function AfterPaintListener(event) {
+        dump("AfterPaintListener\n");
         LogInfo("AfterPaintListener in " + event.target.document.location.href);
         if (event.target.document != currentDoc) {
             // ignore paint events for subframes or old documents in the window.

testing/marionette/driver.js

@@ -31,6 +31,7 @@ Cu.import("chrome://marionette/content/l10n.js");
 Cu.import("chrome://marionette/content/legacyaction.js");
 Cu.import("chrome://marionette/content/modal.js");
 Cu.import("chrome://marionette/content/proxy.js");
+Cu.import("chrome://marionette/content/reftest.js");
 Cu.import("chrome://marionette/content/session.js");
 Cu.import("chrome://marionette/content/wait.js");
 
@@ -3194,6 +3195,65 @@ GeckoDriver.prototype.localizeProperty = function (cmd, resp) {
   resp.body.value = l10n.localizeProperty(urls, id);
 }
 
+/**
+ * Initialize the reftest mode
+ */
+GeckoDriver.prototype.setupReftest = function* (cmd, resp) {
+  if (this._reftest) {
+    throw new UnsupportedOperationError("Called reftest:setup with a reftest session already active");
+  }
+
+  if (this.context !== Context.CHROME) {
+    throw new UnsupportedOperationError("Must set chrome context before running reftests");
+  }
+
+  let {urlCount = {}, screenshot = "unexpected"} = cmd.parameters;
+  if (!["always", "fail", "unexpected"].includes(screenshot)) {
+    throw new InvalidArgumentError("Value of `screenshot` should be 'always', 'fail' or 'unexpected'");
+  }
+
+  this._reftest = new reftest.Runner(this);
+
+  yield this._reftest.setup(urlCount, screenshot);
+};
+
+
+/**
+ * Run a reftest
+ */
+GeckoDriver.prototype.runReftest = function* (cmd, resp) {
+  let {test, references, expected, timeout} = cmd.parameters;
+
+  if (!this._reftest) {
+    throw new UnsupportedOperationError("Called reftest:run before reftest:start");
+  }
+
+  assert.string(test);
+  assert.string(expected);
+  assert.array(references);
+
+  let result = yield this._reftest.run(test, references, expected, timeout);
+
+  resp.body.value = result;
+};
+
+/**
+ * End a reftest run
+ *
+ * Closes the reftest window (without changing the current window handle),
+ * and removes cached canvases.
+ */
+GeckoDriver.prototype.teardownReftest = function* (cmd, resp) {
+  if (!this._reftest) {
+    throw new UnsupportedOperationError("Called reftest:teardown before reftest:start");
+  }
+
+  this._reftest.abort();
+
+  this._reftest = null;
+};
+
+
 GeckoDriver.prototype.commands = {
   // Marionette service
   "Marionette:SetContext": GeckoDriver.prototype.setContext,
@@ -3217,6 +3277,11 @@ GeckoDriver.prototype.commands = {
   "L10n:LocalizeProperty": GeckoDriver.prototype.localizeProperty,
   "localization:l10n:localizeProperty": GeckoDriver.prototype.localizeProperty,  // deprecated, remove in Firefox 60
 
+  // Reftest service
+  "reftest:setup": GeckoDriver.prototype.setupReftest,
+  "reftest:run": GeckoDriver.prototype.runReftest,
+  "reftest:teardown": GeckoDriver.prototype.teardownReftest,
+
   // WebDriver service
   "WebDriver:AcceptDialog": GeckoDriver.prototype.acceptDialog,
   "WebDriver:AddCookie": GeckoDriver.prototype.addCookie,

testing/marionette/jar.mn

@@ -33,6 +33,8 @@ marionette.jar:
   content/transport.js (transport.js)
   content/packets.js (packets.js)
   content/stream-utils.js (stream-utils.js)
+  content/reftest.js (reftest.js)
+  content/reftest.xul (reftest.xul)
 #ifdef ENABLE_TESTS
   content/test.xul (chrome/test.xul)
   content/test2.xul (chrome/test2.xul)

testing/marionette/listener.js

@@ -479,6 +479,7 @@ var multiActionFn = dispatch(multiAction);
 var executeFn = dispatch(execute);
 var executeInSandboxFn = dispatch(executeInSandbox);
 var sendKeysToElementFn = dispatch(sendKeysToElement);
+var reftestWaitFn = dispatch(reftestWait);
 
 /**
  * Start all message listeners
@@ -522,6 +523,7 @@ function startListeners() {
   addMessageListenerId("Marionette:sleepSession", sleepSession);
   addMessageListenerId("Marionette:getAppCacheStatus", getAppCacheStatus);
   addMessageListenerId("Marionette:takeScreenshot", takeScreenshotFn);
+  addMessageListenerId("Marionette:reftestWait", reftestWaitFn);
 }
 
 /**
@@ -1041,14 +1043,15 @@ function waitForPageLoaded(msg) {
  * driver (in chrome space).
  */
 function get(msg) {
-  let {command_id, pageTimeout, url} = msg.json;
-  let loadEventExpected = true;
+  let {command_id, pageTimeout, url, loadEventExpected=null} = msg.json;
 
   try {
     if (typeof url == "string") {
       try {
         let requestedURL = new URL(url).toString();
-        loadEventExpected = navigate.isLoadEventExpected(requestedURL);
+        if (loadEventExpected === null) {
+          loadEventExpected = navigate.isLoadEventExpected(requestedURL);
+        }
       } catch (e) {
         sendError(new InvalidArgumentError("Malformed URL: " + e.message), command_id);
         return;
@@ -1647,5 +1650,116 @@ function takeScreenshot(format, opts = {}) {
   }
 }
 
+function flushRendering() {
+  let content = curContainer.frame;
+  let anyPendingPaintsGeneratedInDescendants = false;
+
+  let windowUtils = content.QueryInterface(Ci.nsIInterfaceRequestor)
+    .getInterface(Ci.nsIDOMWindowUtils);
+
+  function flushWindow(win) {
+    let utils = win.QueryInterface(Ci.nsIInterfaceRequestor)
+      .getInterface(Ci.nsIDOMWindowUtils);
+    let afterPaintWasPending = utils.isMozAfterPaintPending;
+
+    let root = win.document.documentElement;
+    if (root) {
+      try {
+        // Flush pending restyles and reflows for this window
+        root.getBoundingClientRect();
+      } catch (e) {
+        logger.warning(`flushWindow failed: ${e}`);
+      }
+    }
+
+    if (!afterPaintWasPending && utils.isMozAfterPaintPending) {
+      anyPendingPaintsGeneratedInDescendants = true;
+    }
+
+    for (let i = 0; i < win.frames.length; ++i) {
+      flushWindow(win.frames[i]);
+    }
+  }
+  flushWindow(content);
+
+  if (anyPendingPaintsGeneratedInDescendants &&
+      !windowUtils.isMozAfterPaintPending) {
+    logger.error("Internal error: descendant frame generated a MozAfterPaint event, but the root document doesn't have one!");
+  }
+
+  logger.debug(`flushRendering ${windowUtils.isMozAfterPaintPending}`);
+  return windowUtils.isMozAfterPaintPending;
+}
+
+function* reftestWait(url, remote) {
+  let win = curContainer.frame;
+  let document = curContainer.frame.document;
+
+  let windowUtils = content.QueryInterface(Ci.nsIInterfaceRequestor)
+      .getInterface(Ci.nsIDOMWindowUtils);
+
+
+  let reftestWait = false;
+
+  if (document.location.href !== url || document.readyState != "complete") {
+    logger.debug(`Waiting for page load of ${url}`);
+    yield new Promise(resolve => {
+      let maybeResolve = (event) => {
+        if (event.target === curContainer.frame.document &&
+            event.target.location.href === url) {
+          win = curContainer.frame;
+          document = curContainer.frame.document;
+          reftestWait = document.documentElement.classList.contains("reftest-wait");
+          removeEventListener("load", maybeResolve, {once: true});
+          win.setTimeout(resolve, 0);
+        }
+      };
+      addEventListener("load", maybeResolve, true);
+    });
+  } else {
+    // Ensure that the event loop has spun at least once since load,
+    // so that setTimeout(fn, 0) in the load event has run
+    reftestWait = document.documentElement.classList.contains("reftest-wait");
+    yield new Promise(resolve => win.setTimeout(resolve, 0));
+  };
+
+  let root = document.documentElement;
+  if (reftestWait) {
+    // Check again in case reftest-wait was removed since the load event
+    if (root.classList.contains("reftest-wait")) {
+      logger.debug("Waiting for reftest-wait removal");
+      yield new Promise(resolve => {
+        let observer = new win.MutationObserver(() => {
+          if (!root.classList.contains("reftest-wait")) {
+            observer.disconnect();
+            logger.debug("reftest-wait removed");
+            win.setTimeout(resolve, 0);
+          }
+        });
+        observer.observe(root, {attributes: true});
+      });
+    }
+
+    logger.debug("Waiting for rendering");
+
+    yield new Promise(resolve => {
+      let maybeResolve = () => {
+        if (flushRendering()) {
+          win.addEventListener("MozAfterPaint", maybeResolve, {once: true});
+        } else {
+          win.setTimeout(resolve, 0);
+        }
+      };
+      maybeResolve();
+    });
+  } else {
+    flushRendering();
+  }
+
+  if (remote) {
+    windowUtils.updateLayerTree();
+  }
+}
+
 // Call register self when we get loaded
 registerSelf();