Skip to content

Commit

Permalink
Optimize data dependency for blink_web_tests
Browse files Browse the repository at this point in the history 
Some baselines are only useful for certain platforms. Explicitly list
such dependency so that we don't need download unnecessary baselines to
swarming bots. Added a presubmit check to ensure we do not miss any
folders under web_tests.

This should be able to reduce the overhead on the swarming bots. After
this change, we saved 38524 files or directories on Windows.
This is 20.8% of the web_tests folder.

Bug: 1230939
Change-Id: I805f03f9f002a05b0fb1456271aa9b383c9bd3c2
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3139556
Commit-Queue: Weizhong Xia <weizhong@google.com>
Reviewed-by: Xianzhu Wang <wangxianzhu@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@google.com>
Cr-Commit-Position: refs/heads/main@{#918838}
  • Loading branch information
@WeizhongX
WeizhongX authored and Chromium LUCI CQ committed Sep 7, 2021
1 parent 9f4fb15 commit 8f26bc7
Show file tree
Hide file tree
Showing 4 changed files with 193 additions and 3 deletions.
135 changes: 134 additions & 1 deletion BUILD.gn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1240,9 +1240,142 @@ if (!is_ios) {

data_deps = [ ":blink_web_tests_support_data" ]
data = [
"//third_party/blink/web_tests/SmokeTests",
"//third_party/blink/web_tests/VirtualTestSuites",
"//third_party/blink/perf_tests/",
"//third_party/blink/web_tests/",
# List all test expectations here
"//third_party/blink/web_tests/ASANExpectations",
"//third_party/blink/web_tests/LeakExpectations",
"//third_party/blink/web_tests/MSANExpectations",
"//third_party/blink/web_tests/NeverFixTests",
"//third_party/blink/web_tests/SlowTests",
"//third_party/blink/web_tests/StaleTestExpectations",
"//third_party/blink/web_tests/TestExpectations",
"//third_party/blink/web_tests/WebDriverExpectations",
"//third_party/blink/web_tests/WebGPUExpectations",
# === List Test Cases folders here ===
"//third_party/blink/web_tests/accessibility/",
"//third_party/blink/web_tests/android/",
"//third_party/blink/web_tests/animations/",
"//third_party/blink/web_tests/app_banner/",
"//third_party/blink/web_tests/bindings/",
"//third_party/blink/web_tests/clipboard/",
"//third_party/blink/web_tests/compositing/",
"//third_party/blink/web_tests/crypto/",
"//third_party/blink/web_tests/css-parser/",
"//third_party/blink/web_tests/css1/",
"//third_party/blink/web_tests/css2.1/",
"//third_party/blink/web_tests/css3/",
"//third_party/blink/web_tests/cssom/",
"//third_party/blink/web_tests/custom-elements/",
"//third_party/blink/web_tests/custom-properties/",
"//third_party/blink/web_tests/dark-mode/",
"//third_party/blink/web_tests/dom/",
"//third_party/blink/web_tests/editing/",
"//third_party/blink/web_tests/external/",
"//third_party/blink/web_tests/fast/",
"//third_party/blink/web_tests/fonts/",
"//third_party/blink/web_tests/fragmentation/",
"//third_party/blink/web_tests/fullscreen/",
"//third_party/blink/web_tests/gamepad/",
"//third_party/blink/web_tests/harness-tests/",
"//third_party/blink/web_tests/hittesting/",
"//third_party/blink/web_tests/html/",
"//third_party/blink/web_tests/html5lib/",
"//third_party/blink/web_tests/http/",
"//third_party/blink/web_tests/idle-callback/",
"//third_party/blink/web_tests/ietestcenter/",
"//third_party/blink/web_tests/images/",
"//third_party/blink/web_tests/inspector-protocol/",
"//third_party/blink/web_tests/intersection-observer/",
"//third_party/blink/web_tests/jquery/",
"//third_party/blink/web_tests/js/",
"//third_party/blink/web_tests/lifecycle/",
"//third_party/blink/web_tests/loader/",
"//third_party/blink/web_tests/media/",
"//third_party/blink/web_tests/media_capabilities/",
"//third_party/blink/web_tests/mhtml/",
"//third_party/blink/web_tests/navigator_language/",
"//third_party/blink/web_tests/navigator_webdriver/",
"//third_party/blink/web_tests/netinfo/",
"//third_party/blink/web_tests/overflow/",
"//third_party/blink/web_tests/paint/",
"//third_party/blink/web_tests/payments/",
"//third_party/blink/web_tests/permissionclient/",
"//third_party/blink/web_tests/plugins/",
"//third_party/blink/web_tests/pointer-lock/",
"//third_party/blink/web_tests/print_testharness/",
"//third_party/blink/web_tests/printing/",
"//third_party/blink/web_tests/register-protocol-handler/",
"//third_party/blink/web_tests/regress/",
"//third_party/blink/web_tests/resize-observer/",
"//third_party/blink/web_tests/resources/",
"//third_party/blink/web_tests/rootscroller/",
"//third_party/blink/web_tests/screen_orientation/",
"//third_party/blink/web_tests/scrollbars/",
"//third_party/blink/web_tests/scrollingcoordinator/",
"//third_party/blink/web_tests/security/",
"//third_party/blink/web_tests/shadow-dom/",
"//third_party/blink/web_tests/storage/",
"//third_party/blink/web_tests/svg/",
"//third_party/blink/web_tests/synthetic_gestures/",
"//third_party/blink/web_tests/tables/",
"//third_party/blink/web_tests/test_runner/",
"//third_party/blink/web_tests/third_party/",
"//third_party/blink/web_tests/timezonechange/",
"//third_party/blink/web_tests/touchadjustment/",
"//third_party/blink/web_tests/transforms/",
"//third_party/blink/web_tests/transitions/",
"//third_party/blink/web_tests/traversal/",
"//third_party/blink/web_tests/typedcssom/",
"//third_party/blink/web_tests/vibration/",
"//third_party/blink/web_tests/virtual/",
"//third_party/blink/web_tests/virtualkeyboard/",
"//third_party/blink/web_tests/wake-lock/",
"//third_party/blink/web_tests/wasm/",
"//third_party/blink/web_tests/webaudio/",
"//third_party/blink/web_tests/webexposed/",
"//third_party/blink/web_tests/webgpu/",
"//third_party/blink/web_tests/wpt_internal/",
"//third_party/blink/web_tests/xmlviewer/",
# === Test Case Folders Ends ===
]

if (is_win || is_linux || is_fuchsia) {
data += [
"//third_party/blink/web_tests/platform/win/",
"//third_party/blink/web_tests/platform/win7/",
"//third_party/blink/web_tests/platform/win10.1909/",
]
if (is_linux || is_fuchsia) {
data += [
"//third_party/blink/web_tests/platform/linux/",
]
}
if (is_fuchsia) {
data += [
"//third_party/blink/web_tests/platform/fuchsia/",
]
}
} else if (is_mac) {
data += [
"//third_party/blink/web_tests/platform/mac/",
"//third_party/blink/web_tests/platform/mac-mac10.12/",
"//third_party/blink/web_tests/platform/mac-mac10.13/",
"//third_party/blink/web_tests/platform/mac-mac10.14/",
"//third_party/blink/web_tests/platform/mac-mac10.15/",
"//third_party/blink/web_tests/platform/mac-mac-arm11.0/",
]
}

# flag-specific tests only runs on linux as of now
if (is_linux) {
data += [
"//third_party/blink/web_tests/FlagSpecificConfig",
"//third_party/blink/web_tests/FlagExpectations/",
"//third_party/blink/web_tests/flag-specific/",
]
}
}

script_test("devtools_web_tests") {
Expand Down
9 changes: 8 additions & 1 deletion docs/testing/web_tests.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ on this.

There are two ways to run web tests with additional command-line arguments:

### `--flag-specific` or `--additional-driver-flag`:
### --flag-specific or --additional-driver-flag:

```bash
# Actually we prefer --flag-specific in some cases. See below for details.
Expand Down Expand Up @@ -254,6 +254,13 @@ For example, when at least `--additional-driver-flag=--blocking-repaint` and
`--additional-driver-flag=--another-flag` are specified, `short-name` will
be used as name of the flag specific expectation file and the baseline directory.

*** note
[BUILD.gn](../../BUILD.gn) assumes flag-specific builders always runs on linux bots, so
flag-specific test expectations and baselines are only downloaded to linux bots.
If you need run flag-specific builders on other platforms, please update
BUILD.gn to download flag-specific related data to that platform.
***

### Virtual test suites

A *virtual test suite* can be defined in
Expand Down
2 changes: 1 addition & 1 deletion third_party/blink/tools/blinkpy/web_tests/controllers/web_test_finder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ def _read_filter_files(self, filenames, test_path_separator):
except IOError as error:
if error.errno == errno.ENOENT:
_log.critical('')
_log.critical('--test-list file "%s" not found', file)
_log.critical('--test-list file "%s" not found', filename)
raise
return positive_matches, negative_matches, positive_globs, negative_globs

Expand Down
50 changes: 50 additions & 0 deletions third_party/blink/web_tests/PRESUBMIT.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ def _CheckForInvalidPreferenceError(input_api, output_api):
results.append(output_api.PresubmitError('Found an invalid preference %s in expected result %s:%s' % (error.group(1), f, line_num)))
return results


def _CheckRunAfterLayoutAndPaintJS(input_api, output_api):
"""Checks if resources/run-after-layout-and-paint.js and
http/tests/resources/run-after-layout-and-paint.js are the same."""
Expand All @@ -143,6 +144,53 @@ def _CheckRunAfterLayoutAndPaintJS(input_api, output_api):
break
return []


def _CheckForUnlistedTestFolder(input_api, output_api):
"""Checks all the test folders under web_tests are listed in BUILD.gn.
"""
this_dir = input_api.PresubmitLocalPath()
possible_new_dirs = []
for f in input_api.AffectedFiles():
if f.Action() == 'A':
# We only check added folders. For deleted folders, if BUILD.gn is
# not updated, the build will fail at upload step. The reason is that
# we can not know if the folder is deleted as there can be local
# unchecked in files.
path = f.AbsoluteLocalPath()
fns = path[len(this_dir)+1:].split('/')
if len(fns) > 1:
possible_new_dirs.append(fns[0])

if possible_new_dirs:
path_build_gn = input_api.os_path.join(input_api.change.RepositoryRoot(), 'BUILD.gn')
dirs_from_build_gn = []
start_line = '# === List Test Cases folders here ==='
end_line = '# === Test Case Folders Ends ==='
find_start_line = False
for line in input_api.ReadFile(path_build_gn).splitlines():
line = line.strip()
if line.startswith(start_line):
find_start_line = True
continue
if find_start_line:
if line.startswith(end_line):
break
dirs_from_build_gn.append(line.split('/')[-2])
dirs_from_build_gn.extend(['platform', 'FlagExpectations', 'flag-specific'])

new_dirs = [x for x in possible_new_dirs if x not in dirs_from_build_gn]
if new_dirs:
dir_plural = "directories" if len(new_dirs) > 1 else "directory"
error_message = (
'This CL adds new %s(%s) under //third_party/blink/web_tests/, but //BUILD.gn '
'is not updated. Please add the %s to BUILD.gn.' % (dir_plural, ', '.join(new_dirs), dir_plural))
if input_api.is_committing:
return [output_api.PresubmitError(error_message)]
else:
return [output_api.PresubmitPromptWarning(error_message)]
return []


def CheckChangeOnUpload(input_api, output_api):
results = []
results.extend(_CheckTestharnessResults(input_api, output_api))
Expand All @@ -151,6 +199,7 @@ def CheckChangeOnUpload(input_api, output_api):
results.extend(_CheckForJSTest(input_api, output_api))
results.extend(_CheckForInvalidPreferenceError(input_api, output_api))
results.extend(_CheckRunAfterLayoutAndPaintJS(input_api, output_api))
results.extend(_CheckForUnlistedTestFolder(input_api, output_api))
return results


Expand All @@ -159,4 +208,5 @@ def CheckChangeOnCommit(input_api, output_api):
results.extend(_CheckTestharnessResults(input_api, output_api))
results.extend(_CheckFilesUsingEventSender(input_api, output_api))
results.extend(_CheckTestExpectations(input_api, output_api))
results.extend(_CheckForUnlistedTestFolder(input_api, output_api))
return results

0 comments on commit 8f26bc7

Please sign in to comment.