Add MANIFEST.json to .gitignore, regenerate it from template in test …

…runner

This solves 2 problems:

1. Regenerating MANIFEST.json is failing on ToT (bug 697207)
- Now that MANIFEST.json is in .gitignore, regeneration works
2. Regenerating adds MANIFEST.json to MANIFEST.json (bug 683485)
- MANIFEST.json no longer shows up in MANIFEST.json

BUG=697207,666957,683485

Change-Id: I09987b66027e1f94055888c8ead58e7e40896d62
Reviewed-on: https://chromium-review.googlesource.com/447959
Commit-Queue: Jeff Carpenter <jeffcarp@chromium.org>
Reviewed-by: Quinten Yearsley <qyearsley@chromium.org>
Cr-Commit-Position: refs/heads/master@{#454454}

docs/testing/web_platform_tests.md

@@ -92,19 +92,8 @@ Any changes outside of
 upstreamed, and any changes `*-expected.txt`, `OWNERS`, and `MANIFEST.json`,
 will also not be upstreamed.
 
-Note: if you're adding a new test in `external/wpt`, you'll need to re-generate
-MANIFEST.json manually until [CL 2644783003](https://crrev.com/2644783003) is
-landed. The command to do so is:
-
-```bash
-Tools/Scripts/webkitpy/thirdparty/wpt/wpt/manifest --work \
-    --tests-root=LayoutTests/external/wpt
-```
-
-Note: the `--work` argument is important even if you have no local changes.
-Without it the `manifest` script will try to generate a manifest for all files
-in the Chromium tree and take a very long time.
-See [wpt-tools issue #171](https://github.com/w3c/wpt-tools/issues/171).
+Running the layout tests will automatically regenerate MANIFEST.json to pick up
+any local modifications.
 
 Most tests are written using testharness.js, see
 [Writing Layout Tests](./writing_layout_tests.md) and

third_party/WebKit/LayoutTests/external/wpt/.gitignore

@@ -0,0 +1 @@
+MANIFEST.json

third_party/WebKit/Tools/Scripts/webkitpy/common/host_mock.py

@@ -31,6 +31,7 @@
 from webkitpy.common.net.buildbot_mock import MockBuildBot
 from webkitpy.common.net.web_mock import MockWeb
 from webkitpy.common.system.system_host_mock import MockSystemHost
+from webkitpy.common.webkit_finder import WebKitFinder
 
 # New-style ports need to move down into webkitpy.common.
 from webkitpy.layout_tests.builder_list import BuilderList
@@ -54,6 +55,7 @@ def __init__(self,
             time_return_val=time_return_val)
 
         add_unit_tests_to_mock_filesystem(self.filesystem)
+        self._add_base_manifest_to_mock_filesystem(self.filesystem)
         self.web = web or MockWeb()
         self._git = git
 
@@ -74,3 +76,12 @@ def git(self, path=None):
         # Making the checkout_root exist in the mock filesystem makes that chdir not raise.
         self.filesystem.maybe_make_directory(self._git.checkout_root)
         return self._git
+
+    def _add_base_manifest_to_mock_filesystem(self, filesystem):
+        webkit_finder = WebKitFinder(filesystem)
+
+        external_dir = webkit_finder.path_from_webkit_base('LayoutTests', 'external')
+        filesystem.maybe_make_directory(filesystem.join(external_dir, 'wpt'))
+
+        manifest_base_path = filesystem.join(external_dir, 'WPT_BASE_MANIFEST.json')
+        filesystem.files[manifest_base_path] = '{}'

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

@@ -44,6 +44,7 @@
 import time
 
 from webkitpy.common.net.file_uploader import FileUploader
+from webkitpy.common.webkit_finder import WebKitFinder
 from webkitpy.layout_tests.controllers.layout_test_finder import LayoutTestFinder
 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner
 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
@@ -53,6 +54,7 @@
 from webkitpy.layout_tests.models import test_run_results
 from webkitpy.layout_tests.models.test_input import TestInput
 from webkitpy.tool import grammar
+from webkitpy.w3c.wpt_manifest import WPTManifest
 
 _log = logging.getLogger(__name__)
 
@@ -88,13 +90,18 @@ def __init__(self, port, options, printer):
 
         self._results_directory = self._port.results_directory()
         self._finder = LayoutTestFinder(self._port, self._options)
+        self._webkit_finder = WebKitFinder(port.host.filesystem)
         self._runner = LayoutTestRunner(self._options, self._port, self._printer, self._results_directory, self._test_is_slow)
 
     def run(self, args):
         """Run the tests and return a RunDetails object with the results."""
         start_time = time.time()
         self._printer.write_update("Collecting tests ...")
         running_all_tests = False
+
+        # Regenerate MANIFEST.json from template, necessary for web-platform-tests metadata.
+        self._ensure_manifest()
+
         try:
             paths, all_test_names, running_all_tests = self._collect_tests(args)
         except IOError:
@@ -545,3 +552,18 @@ def _worker_number(worker_name):
         for name, value in stats.iteritems():
             json_results_generator.add_path_to_trie(name, value, stats_trie)
         return stats_trie
+
+    def _ensure_manifest(self):
+        fs = self._filesystem
+        external_path = self._webkit_finder.path_from_webkit_base('LayoutTests', 'external')
+        wpt_path = fs.join(external_path, 'wpt')
+        manifest_path = fs.join(external_path, 'wpt', 'MANIFEST.json')
+        base_manifest_path = fs.join(external_path, 'WPT_BASE_MANIFEST.json')
+
+        if not self._filesystem.exists(manifest_path):
+            fs.copyfile(base_manifest_path, manifest_path)
+
+        self._printer.write_update('Generating MANIFEST.json for web-platform-tests ...')
+
+        # TODO(jeffcarp): handle errors
+        WPTManifest.generate_manifest(self._port.host, wpt_path)

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py

@@ -178,3 +178,55 @@ def get_manager():
             if not port.host.filesystem.exists(dir_name):
                 deleted_dir_count = deleted_dir_count + 1
         self.assertEqual(deleted_dir_count, 5)
+
+    # Tests for protected methods - pylint: disable=protected-access
+
+    def test_ensure_manifest_copies_new_manifest(self):
+        host = MockHost()
+        port = host.port_factory.get()
+
+        manifest_path = '/mock-checkout/third_party/WebKit/LayoutTests/external/wpt/MANIFEST.json'
+        self.assertFalse(port.host.filesystem.exists(manifest_path))
+        manager = Manager(port, options=optparse.Values({'max_locked_shards': 1}), printer=FakePrinter())
+        manager._ensure_manifest()
+        self.assertTrue(port.host.filesystem.exists(manifest_path))
+
+        webkit_base = '/mock-checkout/third_party/WebKit'
+        self.assertEqual(
+            port.host.executive.calls,
+            [
+                [
+                    'python',
+                    webkit_base + '/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/manifest',
+                    '--work',
+                    '--tests-root',
+                    webkit_base + '/LayoutTests/external/wpt',
+                ]
+            ]
+        )
+
+    def test_ensure_manifest_updates_manifest_if_it_exists(self):
+        host = MockHost()
+        port = host.port_factory.get('test-mac-mac10.10')
+        manifest_path = '/mock-checkout/third_party/WebKit/LayoutTests/external/wpt/MANIFEST.json'
+
+        host.filesystem.write_binary_file(manifest_path, '{}')
+        self.assertTrue(port.host.filesystem.exists(manifest_path))
+
+        manager = Manager(port, options=optparse.Values({'max_locked_shards': 1}), printer=FakePrinter())
+        manager._ensure_manifest()
+        self.assertTrue(port.host.filesystem.exists(manifest_path))
+
+        webkit_base = '/mock-checkout/third_party/WebKit'
+        self.assertEqual(
+            port.host.executive.calls,
+            [
+                [
+                    'python',
+                    webkit_base + '/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/manifest',
+                    '--work',
+                    '--tests-root',
+                    webkit_base + '/LayoutTests/external/wpt',
+                ]
+            ]
+        )

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

@@ -1086,7 +1086,7 @@ def test_reset_results(self):
             tests_included=True, host=host, new_results=True)
         file_list = host.filesystem.written_files.keys()
         self.assertEqual(details.exit_code, 0)
-        self.assertEqual(len(file_list), 7)
+        self.assertEqual(len(file_list), 8)
         self.assert_baselines(file_list, "passes/image", [".txt", ".png"], err)
 
     def test_missing_results(self):
@@ -1100,7 +1100,7 @@ def test_missing_results(self):
                                       tests_included=True, host=host, new_results=True)
         file_list = host.filesystem.written_files.keys()
         self.assertEqual(details.exit_code, 3)
-        self.assertEqual(len(file_list), 11)
+        self.assertEqual(len(file_list), 12)
         self.assert_baselines(file_list, "failures/unexpected/missing_text", [".txt"], err)
         self.assert_baselines(file_list, "platform/test/failures/unexpected/missing_image", [".png"], err)
         self.assert_baselines(file_list, "platform/test/failures/unexpected/missing_render_tree_dump", [".txt"], err)
@@ -1114,7 +1114,7 @@ def test_new_baseline(self):
             tests_included=True, host=host, new_results=True)
         file_list = host.filesystem.written_files.keys()
         self.assertEqual(details.exit_code, 0)
-        self.assertEqual(len(file_list), 7)
+        self.assertEqual(len(file_list), 8)
         self.assert_baselines(file_list,
                              "platform/test-mac-mac10.10/passes/image", [".txt", ".png"], err)
 
@@ -1125,7 +1125,7 @@ def test_reftest_reset_results(self):
         details, err, _ = logging_run(['--reset-results', 'passes/reftest.html'], tests_included=True, host=host)
         file_list = host.filesystem.written_files.keys()
         self.assertEqual(details.exit_code, 0)
-        self.assertEqual(len(file_list), 5)
+        self.assertEqual(len(file_list), 6)
         self.assert_baselines(file_list, '', [], err)
 
         host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-expected.txt', '')
@@ -1143,7 +1143,7 @@ def test_reftest_new_baseline(self):
         details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'], tests_included=True, host=host)
         file_list = host.filesystem.written_files.keys()
         self.assertEqual(details.exit_code, 0)
-        self.assertEqual(len(file_list), 5)
+        self.assertEqual(len(file_list), 6)
         self.assert_baselines(file_list, '', [], err)
 
         host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-expected.txt', '')

third_party/WebKit/Tools/Scripts/webkitpy/w3c/test_importer_unittest.py

@@ -151,6 +151,7 @@ def test_generate_manifest_successful_run(self):
             host.executive.calls,
             [
                 [
+                    'python',
                     '/mock-checkout/third_party/WebKit/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/manifest',
                     '--work',
                     '--tests-root',

third_party/WebKit/Tools/Scripts/webkitpy/w3c/wpt_manifest.py

@@ -92,8 +92,9 @@ def generate_manifest(host, dest_path):
         """Generates MANIFEST.json on the specified directory."""
         executive = host.executive
         finder = WebKitFinder(host.filesystem)
-        cmd = [finder.path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'thirdparty', 'wpt', 'wpt', 'manifest'),
-               '--work', '--tests-root', dest_path]
+        manifest_exec_path = finder.path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'thirdparty', 'wpt', 'wpt', 'manifest')
+
+        cmd = ['python', manifest_exec_path, '--work', '--tests-root', dest_path]
         _log.debug('Running command: %s', ' '.join(cmd))
         proc = executive.popen(cmd, stdout=executive.PIPE, stderr=executive.PIPE, stdin=executive.PIPE, cwd=finder.webkit_base())
         out, err = proc.communicate('')