diff --git a/content/test/gpu/gpu_tests/gpu_integration_test.py b/content/test/gpu/gpu_tests/gpu_integration_test.py index e32cc538bb4516..86bf44bc538a80 100644 --- a/content/test/gpu/gpu_tests/gpu_integration_test.py +++ b/content/test/gpu/gpu_tests/gpu_integration_test.py @@ -5,6 +5,7 @@ import logging from telemetry.testing import serially_executed_browser_test_case +from telemetry.util import screenshot from gpu_tests import exception_formatter from gpu_tests import gpu_test_expectations @@ -29,22 +30,34 @@ def GenerateTestCases__RunGpuTest(cls, options): for test_name, url, args in cls.GenerateGpuTests(options): yield test_name, (url, test_name, args) - def _RestartBrowser(self, reason): + @classmethod + def StartBrowser(cls): for x in range(0, 3): try: - restart_attempt = ('Restarting browser %d time due to ' - % (x + 1)) - logging.warning(restart_attempt + reason) - self.StopBrowser() - self.SetBrowserOptions(self._finder_options) - self.StartBrowser() - self.tab = self.browser.tabs[0] + restart = 'Starting browser, attempt %d of 3' % (x + 1) + logging.warning(restart) + super(GpuIntegrationTest, cls).StartBrowser() return except Exception: - # If we are on the last try and there is an exception raise it + # If we are on the last try and there is an exception take a screenshot + # to try and capture more about the browser failure and raise if x == 2: + url = screenshot.TryCaptureScreenShotAndUploadToCloudStorage( + cls.platform) + if url is not None: + logging.info("GpuIntegrationTest screenshot of browser failure " + + "located at " + url) + else: + logging.warning("GpuIntegrationTest unable to take screenshot") raise + def _RestartBrowser(self, reason): + logging.warning('Restarting browser due to '+ reason) + self.StopBrowser() + self.SetBrowserOptions(self._finder_options) + self.StartBrowser() + self.tab = self.browser.tabs[0] + def _RunGpuTest(self, url, test_name, args): temp_page = _EmulatedPage(url, test_name) expectations = self.__class__.GetExpectations() diff --git a/content/test/gpu/gpu_tests/gpu_integration_test_unittest.py b/content/test/gpu/gpu_tests/gpu_integration_test_unittest.py index c6231756625195..90f54f92b10bae 100644 --- a/content/test/gpu/gpu_tests/gpu_integration_test_unittest.py +++ b/content/test/gpu/gpu_tests/gpu_integration_test_unittest.py @@ -17,7 +17,6 @@ from gpu_tests import gpu_test_expectations _GLOBAL_TEST_COUNT = 0 -_GLOBAL_RESTART_CRASH = False class SimpleIntegrationUnittest(gpu_integration_test.GpuIntegrationTest): # Must be class-scoped since instances aren't reused across runs. @@ -25,8 +24,6 @@ class SimpleIntegrationUnittest(gpu_integration_test.GpuIntegrationTest): _num_browser_starts = 0 - _num_restart_failures = 0 - @classmethod def Name(cls): return 'simple_integration_unittest' @@ -48,8 +45,8 @@ def setUpClass(cls): finder_options.output_formats = ['none'] finder_options.suppress_gtest_report = True finder_options.output_dir = None - finder_options.upload_bucket = 'public' - finder_options.upload_results = False + finder_options .upload_bucket = 'public' + finder_options .upload_results = False cls._finder_options = finder_options cls.platform = None cls.browser = None @@ -64,9 +61,6 @@ def GenerateGpuTests(cls, options): yield ('expected_skip', 'failure.html', ()) yield ('unexpected_failure', 'failure.html', ()) yield ('unexpected_error', 'error.html', ()) - # This test causes the browser to restart 2 times (max allowed 3) and then - # succeeds on the third attempt - yield ('restart', 'restart.html', ()) @classmethod def _CreateExpectations(cls): @@ -81,26 +75,15 @@ def StartBrowser(cls): super(SimpleIntegrationUnittest, cls).StartBrowser() cls._num_browser_starts += 1 - @classmethod - def StopBrowser(cls): - global _GLOBAL_RESTART_CRASH - if _GLOBAL_RESTART_CRASH: - if cls._num_restart_failures < 2: - cls._num_restart_failures += 1 - raise Exception - else: - _GLOBAL_RESTART_CRASH = False - - super(SimpleIntegrationUnittest, cls).StopBrowser() - - def RunActualGpuTest(self, file_path, *args): if file_path == 'failure.html': self.fail('Expected failure') elif file_path == 'restart.html': - global _GLOBAL_RESTART_CRASH - _GLOBAL_RESTART_CRASH = True - self._RestartBrowser("testing restart on failure") + try: + # This will fail because the browser is already started + self.StartBrowser() + finally: + self.StopBrowser() elif file_path == 'flaky.html': if self.__class__._num_flaky_runs_to_fail > 0: self.__class__._num_flaky_runs_to_fail -= 1 @@ -109,9 +92,73 @@ def RunActualGpuTest(self, file_path, *args): raise Exception('Expected exception') +# TODO(eyaich@): add the actual unittest for start-up retrying logic. +class BrowserStartFailureIntegrationUnittest( + gpu_integration_test.GpuIntegrationTest): + # Must be class-scoped since instances aren't reused across runs. + _num_restart_failures = 0 + + @classmethod + def setUpClass(cls): + finder_options = fakes.CreateBrowserFinderOptions() + finder_options.browser_options.platform = fakes.FakeLinuxPlatform() + finder_options.output_formats = ['none'] + finder_options.suppress_gtest_report = True + finder_options.output_dir = None + finder_options .upload_bucket = 'public' + finder_options .upload_results = False + cls._finder_options = finder_options + cls.platform = None + cls.browser = None + cls.SetBrowserOptions(cls._finder_options) + cls.StartBrowser() + + @classmethod + def _CreateExpectations(cls): + expectations = gpu_test_expectations.GpuTestExpectations() + expectations.Fail('expected_failure') + expectations.Flaky('expected_flaky', max_num_retries=3) + expectations.Skip('expected_skip') + return expectations + + @classmethod + def Name(cls): + return 'browser_start_failure_integration_unittest' + + @classmethod + def GenerateGpuTests(cls, options): + # This test causes the browser to try and restart the browser 3 times. + yield ('restart', 'restart.html', ()) + + def RunActualGpuTest(self, file_path, *args): + if file_path == 'restart.html': + try: + # This will fail because the browser is already started + self.StartBrowser() + finally: + self.StopBrowser() + + class GpuIntegrationTestUnittest(unittest.TestCase): @mock.patch('telemetry.internal.util.binary_manager.InitDependencyManager') def testSimpleIntegrationUnittest(self, mockInitDependencyManager): + self._RunIntegrationTest( + 'simple_integration_unittest', [ + 'expected_failure', + 'setup', + 'unexpected_error', + 'unexpected_failure'], ['expected_flaky']) + # It might be nice to be more precise about the order of operations + # with these browser restarts, but this is at least a start. + self.assertEquals(SimpleIntegrationUnittest._num_browser_starts, 6) + + @mock.patch('telemetry.internal.util.binary_manager.InitDependencyManager') + def testIntegrationUnittestWithBrowserFailure( + self, mockInitDependencyManager): + self._RunIntegrationTest( + 'browser_start_failure_integration_unittest', ['restart'], []) + + def _RunIntegrationTest(self, test_name, failures, successes): options = browser_test_runner.TestRunOptions() # Suppress printing out information for passing tests. options.verbosity = 0 @@ -122,22 +169,13 @@ def testSimpleIntegrationUnittest(self, mockInitDependencyManager): try: browser_test_runner.Run( config, options, - ['simple_integration_unittest', + [test_name, '--write-abbreviated-json-results-to=%s' % temp_file_name]) with open(temp_file_name) as f: test_result = json.load(f) - self.assertEquals(test_result['failures'], [ - 'expected_failure', - 'setup', - 'unexpected_error', - 'unexpected_failure']) - self.assertEquals(test_result['successes'], [ - 'expected_flaky', 'restart']) + self.assertEquals(test_result['failures'], failures) + self.assertEquals(test_result['successes'], successes) self.assertEquals(test_result['valid'], True) - # It might be nice to be more precise about the order of operations - # with these browser restarts, but this is at least a start. - self.assertEquals(SimpleIntegrationUnittest._num_browser_starts, 7) - # Assert that we restarted the browser 2 times due to failure in restart - self.assertEquals(SimpleIntegrationUnittest._num_restart_failures, 2) + finally: os.remove(temp_file_name)