Bug 945222 - Initial import of web-platform-tests testsuite 4/4 : Int…

…egration with build system, mach and mozharness, r=ahal,gps

.gitignore

@@ -61,3 +61,6 @@ GTAGS
 GRTAGS
 GSYMS
 GPATH
+
+# Git clone directory for updating web-platform-tests
+testing/web-platform/sync/

.hgignore

@@ -75,3 +75,6 @@ GPATH
 # Unit tests for Loop
 ^browser/components/loop/standalone/content/config\.js$
 ^browser/components/loop/standalone/node_modules/
+
+# Git clone directory for updating web-platform-tests
+^testing/web-platform/sync/

build/mach_bootstrap.py

@@ -43,6 +43,8 @@
     'xpcom/idl-parser',
     'testing',
     'testing/xpcshell',
+    'testing/web-platform',
+    'testing/web-platform/harness',
     'testing/marionette/client',
     'testing/marionette/client/marionette',
     'testing/marionette/transport',
@@ -80,6 +82,7 @@
     'testing/mochitest/mach_commands.py',
     'testing/xpcshell/mach_commands.py',
     'testing/talos/mach_commands.py',
+    'testing/web-platform/mach_commands.py',
     'testing/xpcshell/mach_commands.py',
     'tools/docs/mach_commands.py',
     'tools/mercurial/mach_commands.py',

testing/config/mozharness/web_platform_tests_config.py

@@ -0,0 +1,11 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+config = {
+    "options": [
+        "--prefs-root=%(test_path)s/prefs",
+        "--processes=1",
+        "--config=%(test_path)s/wptrunner.ini",
+    ],
+}

testing/profiles/Makefile.in

@@ -1,20 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-include $(topsrcdir)/config/rules.mk
-
 MOCHITEST_PROFILE_FILES = \
   prefs_general.js \
   prefs_b2g_unittest.js \
   webapps_mochitest.json \
   $(NULL)
+WPT_PROFILE_FILES = $(MOCHITEST_PROFILE_FILES)
 
-_DEST_DIR = $(DEPTH)/_tests/testing/mochitest/profile_data
-libs:: $(MOCHITEST_PROFILE_FILES)
-	$(PYTHON) $(topsrcdir)/config/nsinstall.py $^ $(_DEST_DIR)
+MOCHITEST_PROFILE_DEST = $(DEPTH)/_tests/testing/mochitest/profile_data
+WPT_PROFILE_DEST = $(DEPTH)/_tests/web-platform/prefs
 
-stage-package: PKG_STAGE = $(DIST)/test-stage
-stage-package:
-	$(NSINSTALL) -D $(PKG_STAGE)/
-	@(cd $(srcdir) && tar $(TAR_CREATE_FLAGS) - $(MOCHITEST_PROFILE_FILES)) | (cd $(PKG_STAGE)/mochitest/profile_data && tar -xf -)
+INSTALL_TARGETS += MOCHITEST_PROFILE
+INSTALL_TARGETS += WPT_PROFILE

testing/testsuite-targets.mk

@@ -403,6 +403,7 @@ package-tests: \
   stage-cppunittests \
   stage-jittest \
   stage-steeplechase \
+  stage-web-platform-tests \
   $(NULL)
 else
 # This staging area has been built for us by universal/flight.mk
@@ -547,6 +548,10 @@ stage-marionette: make-stage-dir
 
 stage-mozbase: make-stage-dir
 	$(MAKE) -C $(DEPTH)/testing/mozbase stage-package
+
+stage-web-platform-tests: make-stage-dir
+	$(MAKE) -C $(DEPTH)/testing/web-platform stage-package
+
 .PHONY: \
   mochitest \
   mochitest-plain \
@@ -573,5 +578,6 @@ stage-mozbase: make-stage-dir
   stage-modules \
   stage-marionette \
   stage-steeplechase \
+  stage-web-platform-tests \
   $(NULL)
 

testing/web-platform/Makefile.in

@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+WPT_TESTS = \
+  tests \
+  $(NULL)
+
+WPT_METADATA = \
+  meta \
+  $(NULL)
+
+
+_DEST_DIR = $(DEPTH)/_tests/web-platform
+PKG_STAGE = $(DIST)/test-stage
+
+WEBPLATFORM_FILES = \
+  runtests.py \
+  wptrunner.ini \
+  $(NULL)
+WEBPLATFORM_DEST = $(_DEST_DIR)
+INSTALL_TARGETS += WEBPLATFORM
+
+libs::
+	$(INSTALL) $(topsrcdir)/testing/web-platform/harness/wptrunner $(_DEST_DIR)
+libs:: $(WPT_TESTS)
+	$(INSTALL) $(foreach f,$^,"$f") $(_DEST_DIR)
+libs:: $(WPT_METADATA)
+	$(INSTALL) $(foreach f,$^,"$f") $(_DEST_DIR)
+
+stage-package:
+	$(NSINSTALL) -D $(PKG_STAGE)/web-platform
+	@cp $(DEPTH)/mozinfo.json $(PKG_STAGE)/web-platform
+	@(cd $(DEPTH)/_tests/ && tar $(TAR_CREATE_FLAGS) - web-platform) | (cd $(PKG_STAGE) && tar -xf -)

testing/web-platform/README.md

@@ -0,0 +1,193 @@
+web-platform-tests
+==================
+
+This directory contains the W3C
+[web-platform-tests](http://github.com/w3c/web-platform-tests). They
+can be run using `mach`:
+
+    mach web-platform-tests
+
+To limit the testrun to certain directories use the `--include` option;
+for example:
+
+    mach web-platform-tests --include=dom
+
+The testsuite contains a mix of javascript tests and reftests. To
+limit the type of tests that get run, use `--test-type=testharness` for
+javascript tests or `--test-type=reftest` for reftests.
+
+FAQ
+---
+
+* I fixed a bug and some tests have started to pass. How do I fix the
+  UNEXPECTED-PASS messages when web-platform-tests is run?
+
+  You need to update the expectation data for those tests. See the
+  section on expectations below.
+
+* I want to write some new tests for the web-platform-tests
+  testsuite. How do I do that?
+
+  See the section on tests below. At the moment you will have to
+  submit the tests directly to the W3C and wait for them to be
+  imported into the Mozilla tree.
+
+* A test is unstable; how do I disable it?
+
+  See the section on disabling tests.
+
+Directories
+-----------
+
+`tests/` contains the tests themselves. This is a copy of a certain
+revision of web-platform-tests. The contents of this directory must
+not be modified locally as any modifications will be overwritten
+whenever a new upstream sync is performed.
+
+`harness/` contains the [wptrunner](http://github.com/w3c/wptrunner)
+test runner. Again the contents of this directory will be overwritten
+on update.
+
+`meta/` contains Gecko-specific expectation data. This is explained in
+the following section.
+
+Expectation Data
+----------------
+
+With the tests coming from upstream, it is not guaranteed that they
+all pass in Gecko-based browsers. For this reason it is necessary to
+provide metadata about the expected results of each test. This is
+provided in a set of manifest files in the `meta/` subdirectories.
+
+There is one manifest file per test with "non-default"
+expectations. By default tests are expected to PASS, and tests with
+subtests are expected to have an overall status of OK. The manifest
+file of a test has the same path as the test file but under the `meta`
+directory rather than the `tests` directory and has the suffix `.ini`.
+
+The format of these files is similar to `ini` files, but with a couple
+of important differences; sections can be nested using indentation,
+and only `:` is permitted as a key-value separator. For example the
+expectation file for a test with one failing subtest and one erroring
+subtest might look like:
+
+    [filename.html]
+        type: testharness
+
+        [Subtest name for failing test]
+            expected: FAIL
+
+        [Subtest name for erroring test]
+            expected: ERROR
+
+Expectations can also be made platform-specific using a simple
+python-like conditional syntax e.g. for a test that times out on linux
+but otherwise fails:
+
+    [filename.html]
+        type: reftest
+        expected:
+            if os == "linux": TIMEOUT
+            FAIL
+
+The available variables for the conditions are those provided by
+[mozinfo](http://mozbase.readthedocs.org/en/latest/mozinfo.html).
+
+For more information on manifest files, see the
+[wptrunner documentation](http://wptrunner.readthedocs.org/en/latest/expectation.html).
+
+Autogenerating Expectation Data
+-------------------------------
+
+After changing some code it may be necessary to update the expectation
+data for the relevant tests. This can of course be done manually, but
+tools are available to automate much of the process.
+
+First one must run the tests that have changed status, and save the
+raw log output to a file:
+
+    mach web-platform-tests --include=url/of/test.html --log-raw=new_results.log
+
+Then the `web-platform-tests-update` command may be run using this log
+data to update the expectation files:
+
+    mach web-platform-tests-update --no-check-clean new_results.log
+
+By default this only updates the results data for the current
+platform. To forcibly overwrite all existing result data, use the
+`--ignore-existing` option to the update command.
+
+Disabling Tests
+---------------
+
+Tests are disabled using the same manifest files used to set
+expectation values. For example, if a test is unstable on Windows, it
+can be disabled using an ini file with the contents:
+
+    [filename.html]
+        type: testharness
+        disabled:
+            if os == "win": https://bugzilla.mozilla.org/show_bug.cgi?id=1234567
+
+Test Format
+-----------
+
+Javascript tests are written using
+[testharness.js](http://github.com/w3c/testharness.js/). Reftests are
+similar to standard Gecko reftests without an explicit manifest file,
+but with in-test or filename conventions for identifying the
+reference.
+
+New tests must presently be submitted upstream before they can be run
+on Mozilla infrastructure. This situation is expected to be temporary.
+
+Full documentation on test authoring and submission can be found on
+[testthewebforward.org](http://testthewebforward.org/docs).
+
+Running Tests In Other Browsers
+-------------------------------
+
+web-platform-tests is cross browser, and the runner is compatible with
+multiple browsers. Therefore it's possible to check the behaviour of
+tests in other browsers. This is somewhat more involved than running
+them in Firefox since extra dependencies may be required. For example
+to test in Chrome:
+
+1. Download the chromedriver binary and place it somewhere sensible
+   e.g. `~/bin`
+
+2. In your gecko source tree activate the virtualenv created by mach,
+   since this has most dependencies already installed. This is typically
+   in objdir/_virtualenv and is activated via e.g.
+
+        source objdir/_virtualenv/bin/activate
+
+3. Install the extra requirements:
+
+        cd testing/web-platform/harness
+        pip install -r requirements_chrome.txt
+
+4. Edit the config file `testing/web-platform/wptrunner.ini` so that
+   Chrome support is enabled by changing the section that reads:
+
+        [products]
+        firefox =
+
+   to read
+
+        [products]
+        firefox =
+        chrome =
+
+   (alternatively create a new config file elsewhere and use the
+   `--config` option to `runtests.py` to point wptrunner at this config
+   file).
+
+5. Run `runtests.py` using the location of chromedriver as
+   the binary:
+
+        cd testing/web-platform
+        python runtests.py --product=chrome --binary=~/bin/chromedriver --log-mach=-
+
+By default this will use the same test checkout and metadata as are in
+the Gecko tree, so it's easy to compare behaviour relative to Firefox.