diff --git a/components/cronet/android/test/javaperftests/run.py b/components/cronet/android/test/javaperftests/run.py index afb63e683043a0..e0556e6447978a 100755 --- a/components/cronet/android/test/javaperftests/run.py +++ b/components/cronet/android/test/javaperftests/run.py @@ -206,10 +206,10 @@ def Measure(self, platform, results): 'ms', jsonResults[test])) -@benchmark.Enabled('android') class CronetPerfTestBenchmark(benchmark.Benchmark): # Benchmark implementation spawning off Cronet perf test measurement and # StorySet. + SUPPORTED_PLATFORMS = [story.expectations.ALL_ANDROID] def __init__(self, max_failures=None): super(CronetPerfTestBenchmark, self).__init__(max_failures) @@ -221,6 +221,12 @@ def CreatePageTest(self, options): def CreateStorySet(self, options): return CronetPerfTestStorySet(self._device) + def GetExpectations(self): + class StoryExpectations(story.expectations.StoryExpectations): + def SetExpectations(self): + pass # Nothing disabled. + return StoryExpectations() + class QuicServer(object): diff --git a/tools/chrome_proxy/integration_tests/chrome_proxy_benchmark.py b/tools/chrome_proxy/integration_tests/chrome_proxy_benchmark.py index df48a6268aad84..7b58bcf78501e3 100644 --- a/tools/chrome_proxy/integration_tests/chrome_proxy_benchmark.py +++ b/tools/chrome_proxy/integration_tests/chrome_proxy_benchmark.py @@ -6,6 +6,7 @@ from integration_tests import chrome_proxy_measurements as measurements from integration_tests import chrome_proxy_pagesets as pagesets from telemetry import benchmark +from telemetry import decorators DESKTOP_PLATFORMS = ['mac', 'linux', 'win', 'chromeos'] WEBVIEW_PLATFORMS = ['android-webview', 'android-webview-instrumentation'] @@ -49,7 +50,7 @@ def Name(cls): return 'chrome_proxy_benchmark.client_type.client_type' -@benchmark.Disabled(*WEBVIEW_PLATFORMS) +@decorators.Disabled(*WEBVIEW_PLATFORMS) class ChromeProxyLoFi(ChromeProxyBenchmark): tag = 'lo_fi' test = measurements.ChromeProxyLoFi @@ -60,7 +61,7 @@ def Name(cls): return 'chrome_proxy_benchmark.lo_fi.lo_fi' -@benchmark.Disabled(*WEBVIEW_PLATFORMS) +@decorators.Disabled(*WEBVIEW_PLATFORMS) class ChromeProxyCacheLoFiDisabled(ChromeProxyBenchmark): tag = 'cache_lo_fi_disabled' test = measurements.ChromeProxyCacheLoFiDisabled @@ -71,7 +72,7 @@ def Name(cls): return 'chrome_proxy_benchmark.lo_fi.cache_lo_fi_disabled' -@benchmark.Disabled(*WEBVIEW_PLATFORMS) +@decorators.Disabled(*WEBVIEW_PLATFORMS) class ChromeProxyCacheProxyDisabled(ChromeProxyBenchmark): tag = 'cache_proxy_disabled' test = measurements.ChromeProxyCacheProxyDisabled @@ -82,7 +83,7 @@ def Name(cls): return 'chrome_proxy_benchmark.lo_fi.cache_proxy_disabled' -@benchmark.Disabled(*WEBVIEW_PLATFORMS) +@decorators.Disabled(*WEBVIEW_PLATFORMS) class ChromeProxyLitePage(ChromeProxyBenchmark): tag = 'lite_page' test = measurements.ChromeProxyLitePage @@ -143,7 +144,7 @@ def Name(cls): return 'chrome_proxy_benchmark.html5test.html5test' -@benchmark.Enabled(*DESKTOP_PLATFORMS) +@decorators.Enabled(*DESKTOP_PLATFORMS) class ChromeProxyYouTube(ChromeProxyBenchmark): tag = 'youtube' test = measurements.ChromeProxyYouTube @@ -174,7 +175,7 @@ def Name(cls): return 'chrome_proxy_benchmark.block_once.block_once' -@benchmark.Disabled(*(DESKTOP_PLATFORMS + WEBVIEW_PLATFORMS)) +@decorators.Disabled(*(DESKTOP_PLATFORMS + WEBVIEW_PLATFORMS)) # Safebrowsing is enabled for Android and iOS. class ChromeProxySafeBrowsingOn(ChromeProxyBenchmark): tag = 'safebrowsing_on' @@ -191,7 +192,7 @@ def Name(cls): return 'chrome_proxy_benchmark.safebrowsing_on.safebrowsing' -@benchmark.Enabled(*(DESKTOP_PLATFORMS + WEBVIEW_PLATFORMS)) +@decorators.Enabled(*(DESKTOP_PLATFORMS + WEBVIEW_PLATFORMS)) # Safebrowsing is switched off for Android Webview and all desktop platforms. class ChromeProxySafeBrowsingOff(ChromeProxyBenchmark): tag = 'safebrowsing_off' @@ -283,7 +284,7 @@ def Name(cls): return 'chrome_proxy_benchmark.client_config.synthetic' -@benchmark.Enabled(*DESKTOP_PLATFORMS) +@decorators.Enabled(*DESKTOP_PLATFORMS) class ChromeProxyVideoDirect(benchmark.Benchmark): tag = 'video' test = measurements.ChromeProxyVideoValidation @@ -294,7 +295,7 @@ def Name(cls): return 'chrome_proxy_benchmark.video.direct' -@benchmark.Enabled(*DESKTOP_PLATFORMS) +@decorators.Enabled(*DESKTOP_PLATFORMS) class ChromeProxyVideoProxied(benchmark.Benchmark): tag = 'video' test = measurements.ChromeProxyVideoValidation @@ -305,7 +306,7 @@ def Name(cls): return 'chrome_proxy_benchmark.video.proxied' -@benchmark.Enabled(*DESKTOP_PLATFORMS) +@decorators.Enabled(*DESKTOP_PLATFORMS) class ChromeProxyVideoCompare(benchmark.Benchmark): """Comparison of direct and proxied video fetches. @@ -321,7 +322,7 @@ class ChromeProxyVideoCompare(benchmark.Benchmark): def Name(cls): return 'chrome_proxy_benchmark.video.compare' -@benchmark.Enabled(*DESKTOP_PLATFORMS) +@decorators.Enabled(*DESKTOP_PLATFORMS) class ChromeProxyVideoFrames(benchmark.Benchmark): """Check for video frames similar to original video.""" @@ -333,7 +334,7 @@ class ChromeProxyVideoFrames(benchmark.Benchmark): def Name(cls): return 'chrome_proxy_benchmark.video.frames' -@benchmark.Enabled(*DESKTOP_PLATFORMS) +@decorators.Enabled(*DESKTOP_PLATFORMS) class ChromeProxyVideoAudio(benchmark.Benchmark): """Check that audio is similar to original video.""" diff --git a/tools/perf/benchmarks/dromaeo.py b/tools/perf/benchmarks/dromaeo.py index b14af1872b5463..248fbfa08ccd5a 100644 --- a/tools/perf/benchmarks/dromaeo.py +++ b/tools/perf/benchmarks/dromaeo.py @@ -9,7 +9,6 @@ from core import perf_benchmark from telemetry import benchmark -from telemetry import decorators from telemetry import page as page_module from telemetry.page import legacy_page_test from telemetry import story @@ -191,8 +190,6 @@ def SetExpectations(self): return StoryExpectations() -# Flaky on Windows: https://crbug.com/763263 -@decorators.Disabled('win') @benchmark.Owner(emails=['jbroman@chromium.org', 'yukishiino@chromium.org', 'haraken@chromium.org']) @@ -211,5 +208,5 @@ def Name(cls): def GetExpectations(self): class StoryExpectations(story.expectations.StoryExpectations): def SetExpectations(self): - pass # http://dromaeo.com?dom-traverse not disabled. + self.DisableBenchmark([story.expectations.ALL_WIN], 'crbug.com/763263') return StoryExpectations() diff --git a/tools/perf/contrib/cros_benchmarks/page_cycler_v2.py b/tools/perf/contrib/cros_benchmarks/page_cycler_v2.py index f2f60516bcd9ca..bfaf815829759b 100644 --- a/tools/perf/contrib/cros_benchmarks/page_cycler_v2.py +++ b/tools/perf/contrib/cros_benchmarks/page_cycler_v2.py @@ -40,8 +40,6 @@ def ShouldDisable(cls, possible_browser): return False -@benchmark.Disabled('win10') -@benchmark.Disabled('android') # crbug.com/654217 @benchmark.Owner(emails=['kouhei@chromium.org', 'ksakamoto@chromium.org']) class PageCyclerV2Typical25(_PageCyclerV2): """Page load time benchmark for a 25 typical web pages. @@ -62,7 +60,7 @@ def CreateStorySet(self, options): def GetExpectations(self): class StoryExpectations(story.expectations.StoryExpectations): def SetExpectations(self): - pass # Nothing disabled. + pass # Nothing disabled. return StoryExpectations() @@ -85,7 +83,7 @@ def CreateStorySet(self, options): def GetExpectations(self): class StoryExpectations(story.expectations.StoryExpectations): def SetExpectations(self): - pass # Nothing disabled. + pass # Nothing disabled. return StoryExpectations() @@ -108,7 +106,7 @@ def CreateStorySet(self, options): def GetExpectations(self): class StoryExpectations(story.expectations.StoryExpectations): def SetExpectations(self): - pass # Nothing disabled. + pass # Nothing disabled. return StoryExpectations() @@ -131,11 +129,10 @@ def CreateStorySet(self, options): def GetExpectations(self): class StoryExpectations(story.expectations.StoryExpectations): def SetExpectations(self): - pass # Nothing disabled. + pass # Nothing disabled. return StoryExpectations() -@benchmark.Disabled('android') # crbug.com/666898 @benchmark.Owner(emails=['kouhei@chromium.org', 'ksakamoto@chromium.org']) class PageCyclerV2IntlJaZh(_PageCyclerV2): """Page load time benchmark for a variety of pages in Japanese and Chinese. @@ -154,7 +151,7 @@ def CreateStorySet(self, options): def GetExpectations(self): class StoryExpectations(story.expectations.StoryExpectations): def SetExpectations(self): - pass # Nothing disabled. + pass # Nothing disabled. return StoryExpectations() @@ -177,5 +174,5 @@ def CreateStorySet(self, options): def GetExpectations(self): class StoryExpectations(story.expectations.StoryExpectations): def SetExpectations(self): - pass # Nothing disabled. + pass # Nothing disabled. return StoryExpectations() diff --git a/tools/perf/contrib/cros_benchmarks/tab_switching_bench.py b/tools/perf/contrib/cros_benchmarks/tab_switching_bench.py index e09eeeea37e9ef..ae4326b5187ba9 100644 --- a/tools/perf/contrib/cros_benchmarks/tab_switching_bench.py +++ b/tools/perf/contrib/cros_benchmarks/tab_switching_bench.py @@ -15,7 +15,6 @@ @benchmark.Owner(emails=['vovoy@chromium.org'], component='OS>Performance') -@benchmark.Enabled('chromeos') class CrosTabSwitchingTypical24(perf_benchmark.PerfBenchmark): """Measures tab switching performance with 24 tabs. @@ -31,6 +30,8 @@ class CrosTabSwitchingTypical24(perf_benchmark.PerfBenchmark): cros_tab_switching.typical_24 --tabset-repeat=5 """ test = tab_switching_measure.CrosTabSwitchingMeasurement + SUPPORTED_PLATFORMS = [story.expectations.ALL_CHROMEOS] + @classmethod def AddBenchmarkCommandLineArgs(cls, parser): @@ -56,5 +57,5 @@ def Name(cls): def GetExpectations(self): class StoryExpectations(story.expectations.StoryExpectations): def SetExpectations(self): - pass # No tests disabled. + pass # Nothing disabled. return StoryExpectations() diff --git a/tools/perf/contrib/oilpan/oilpan_benchmarks.py b/tools/perf/contrib/oilpan/oilpan_benchmarks.py index 354a179e54d78c..bed8b9a8477b88 100644 --- a/tools/perf/contrib/oilpan/oilpan_benchmarks.py +++ b/tools/perf/contrib/oilpan/oilpan_benchmarks.py @@ -16,7 +16,7 @@ from contrib.oilpan import oilpan_gc_times -@benchmark.Enabled('content-shell') + class OilpanGCTimesBlinkPerfStress(perf_benchmark.PerfBenchmark): tag = 'blink_perf_stress' test = oilpan_gc_times.OilpanGCTimesForInternals @@ -52,10 +52,10 @@ def SetExpectations(self): return StoryExpectations() -@benchmark.Enabled('android') class OilpanGCTimesKeySilkCases(perf_benchmark.PerfBenchmark): test = oilpan_gc_times.OilpanGCTimesForSmoothness page_set = page_sets.KeySilkCasesPageSet + SUPPORTED_PLATFORMS = [story.expectations.ALL_ANDROID] @classmethod def Name(cls): @@ -73,11 +73,11 @@ def SetExpectations(self): return StoryExpectations() -@benchmark.Enabled('android') class OilpanGCTimesSyncScrollKeyMobileSites(perf_benchmark.PerfBenchmark): tag = 'sync_scroll' test = oilpan_gc_times.OilpanGCTimesForSmoothness page_set = page_sets.KeyMobileSitesSmoothPageSet + SUPPORTED_PLATFORMS = [story.expectations.ALL_ANDROID] def SetExtraBrowserOptions(self, options): silk_flags.CustomizeBrowserOptionsForSyncScrolling(options) diff --git a/tools/perf/core/trybot_command_unittest.py b/tools/perf/core/trybot_command_unittest.py index 3980ca7736c3e1..733a31cf5a6658 100644 --- a/tools/perf/core/trybot_command_unittest.py +++ b/tools/perf/core/trybot_command_unittest.py @@ -16,7 +16,7 @@ from core import trybot_command import mock from telemetry import benchmark - +from telemetry import decorators class FakeProcess(object): @@ -911,6 +911,7 @@ def testAttemptTryjobWithDepsRevisionArg(self): self.assertEquals(output, sys.stdout.getvalue().strip()) +# TODO(rnephew): Modernize these tests to use StoryExpectations. class IsBenchmarkDisabledOnTrybotPlatformTest(unittest.TestCase): def IsBenchmarkDisabled(self, benchmark_class, trybot_name): @@ -918,7 +919,7 @@ def IsBenchmarkDisabled(self, benchmark_class, trybot_name): benchmark_class, trybot_name)[0] def testBenchmarkIsDisabledAll(self): - @benchmark.Disabled('all') + @decorators.Disabled('all') class FooBenchmark(benchmark.Benchmark): pass self.assertTrue(self.IsBenchmarkDisabled(FooBenchmark, 'all')) @@ -928,7 +929,7 @@ class FooBenchmark(benchmark.Benchmark): self.assertTrue(self.IsBenchmarkDisabled(FooBenchmark, 'winx64ati')) def testBenchmarkIsEnabledAll(self): - @benchmark.Enabled('all') + @decorators.Enabled('all') class FooBenchmark(benchmark.Benchmark): pass self.assertFalse(self.IsBenchmarkDisabled(FooBenchmark, 'all')) @@ -938,7 +939,7 @@ class FooBenchmark(benchmark.Benchmark): self.assertFalse(self.IsBenchmarkDisabled(FooBenchmark, 'winx64ati')) def testBenchmarkIsDisabledOnMultiplePlatforms(self): - @benchmark.Disabled('win', 'mac') + @decorators.Disabled('win', 'mac') class FooBenchmark(benchmark.Benchmark): pass self.assertFalse(self.IsBenchmarkDisabled(FooBenchmark, 'all')) @@ -949,7 +950,7 @@ class FooBenchmark(benchmark.Benchmark): self.assertTrue(self.IsBenchmarkDisabled(FooBenchmark, 'winx64ati')) def testBenchmarkIsEnabledOnMultiplePlatforms(self): - @benchmark.Enabled('win', 'mac') + @decorators.Enabled('win', 'mac') class FooBenchmark(benchmark.Benchmark): pass self.assertFalse(self.IsBenchmarkDisabled(FooBenchmark, 'all')) diff --git a/tools/perf/measurements/clock_domain_test.py b/tools/perf/measurements/clock_domain_test.py index b28cfc5c543a46..bc03404f75b9dc 100644 --- a/tools/perf/measurements/clock_domain_test.py +++ b/tools/perf/measurements/clock_domain_test.py @@ -2,7 +2,6 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from telemetry import benchmark from telemetry import decorators from telemetry.testing import tab_test_case from telemetry.timeline import tracing_config @@ -17,7 +16,7 @@ class ClockDomainTest(tab_test_case.TabTestCase): # Don't run this test on Android; it's not supposed to work on Android # (since when doing Android tracing there are two different devices, # so the clock domains will be different) - @benchmark.Disabled('android') + @decorators.Disabled('android') @decorators.Isolated def testTelemetryUsesChromeClockDomain(self): diff --git a/tools/perf/measurements/tab_switching_unittest.py b/tools/perf/measurements/tab_switching_unittest.py index 1e130e42dc06e3..d882553b431eca 100644 --- a/tools/perf/measurements/tab_switching_unittest.py +++ b/tools/perf/measurements/tab_switching_unittest.py @@ -6,7 +6,7 @@ from measurements import tab_switching import mock from page_sets.system_health import multi_tab_stories -from telemetry import benchmark +from telemetry import decorators from telemetry import story as story_module from telemetry.internal.results import page_test_results from telemetry.testing import page_test_test_case @@ -95,9 +95,9 @@ def testIsDone(self): [browser.tabs[-1]] * 2] self.assertEqual(expected_calls, mock_get_histogram.mock_calls) - @benchmark.Enabled('has tabs') - @benchmark.Disabled('mac') - @benchmark.Disabled('android') + @decorators.Enabled('has tabs') + @decorators.Disabled('mac') + @decorators.Disabled('android') def testTabSwitching(self): """IT of TabSwitching measurement and multi-tab story""" ps = story_module.StorySet()