[content/test/gpu] Pushing the restart logic into start browser.

This patch is done on behalf of eyaich@chromium.org (original work in
https://codereview.chromium.org/2209673003).

For reviewing this CL, the 1st patch set is the patch set 1 of
https://codereview.chromium.org/2209673003.
The 2nd patch set address some nits & add TODO for improve unittest logic.

BUG=chromium:628022
CQ_INCLUDE_TRYBOTS=master.tryserver.chromium.linux:linux_optional_gpu_tests_rel;master.tryserver.chromium.mac:mac_optional_gpu_tests_rel;master.tryserver.chromium.win:win_optional_gpu_tests_rel

Review-Url: https://codereview.chromium.org/2219593003
Cr-Commit-Position: refs/heads/master@{#409971}

content/test/gpu/gpu_tests/gpu_integration_test.py

@@ -5,6 +5,7 @@
 import logging
 
 from telemetry.testing import serially_executed_browser_test_case
+from telemetry.util import screenshot
 
 from gpu_tests import exception_formatter
 from gpu_tests import gpu_test_expectations
@@ -29,22 +30,34 @@ def GenerateTestCases__RunGpuTest(cls, options):
     for test_name, url, args in cls.GenerateGpuTests(options):
       yield test_name, (url, test_name, args)
 
-  def _RestartBrowser(self, reason):
+  @classmethod
+  def StartBrowser(cls):
     for x in range(0, 3):
       try:
-        restart_attempt = ('Restarting browser %d time due to '
-          % (x + 1))
-        logging.warning(restart_attempt + reason)
-        self.StopBrowser()
-        self.SetBrowserOptions(self._finder_options)
-        self.StartBrowser()
-        self.tab = self.browser.tabs[0]
+        restart = 'Starting browser, attempt %d of 3' % (x + 1)
+        logging.warning(restart)
+        super(GpuIntegrationTest, cls).StartBrowser()
         return
       except Exception:
-        # If we are on the last try and there is an exception raise it
+        # If we are on the last try and there is an exception take a screenshot
+        # to try and capture more about the browser failure and raise
         if x == 2:
+          url = screenshot.TryCaptureScreenShotAndUploadToCloudStorage(
+            cls.platform)
+          if url is not None:
+            logging.info("GpuIntegrationTest screenshot of browser failure " +
+              "located at " + url)
+          else:
+            logging.warning("GpuIntegrationTest unable to take screenshot")
           raise
 
+  def _RestartBrowser(self, reason):
+    logging.warning('Restarting browser due to '+ reason)
+    self.StopBrowser()
+    self.SetBrowserOptions(self._finder_options)
+    self.StartBrowser()
+    self.tab = self.browser.tabs[0]
+
   def _RunGpuTest(self, url, test_name, args):
     temp_page = _EmulatedPage(url, test_name)
     expectations = self.__class__.GetExpectations()

content/test/gpu/gpu_tests/gpu_integration_test_unittest.py

@@ -17,16 +17,13 @@
 from gpu_tests import gpu_test_expectations
 
 _GLOBAL_TEST_COUNT = 0
-_GLOBAL_RESTART_CRASH = False
 
 class SimpleIntegrationUnittest(gpu_integration_test.GpuIntegrationTest):
   # Must be class-scoped since instances aren't reused across runs.
   _num_flaky_runs_to_fail = 2
 
   _num_browser_starts = 0
 
-  _num_restart_failures = 0
-
   @classmethod
   def Name(cls):
     return 'simple_integration_unittest'
@@ -48,8 +45,8 @@ def setUpClass(cls):
     finder_options.output_formats = ['none']
     finder_options.suppress_gtest_report = True
     finder_options.output_dir = None
-    finder_options.upload_bucket = 'public'
-    finder_options.upload_results = False
+    finder_options .upload_bucket = 'public'
+    finder_options .upload_results = False
     cls._finder_options = finder_options
     cls.platform = None
     cls.browser = None
@@ -64,9 +61,6 @@ def GenerateGpuTests(cls, options):
     yield ('expected_skip', 'failure.html', ())
     yield ('unexpected_failure', 'failure.html', ())
     yield ('unexpected_error', 'error.html', ())
-    # This test causes the browser to restart 2 times (max allowed 3) and then
-    # succeeds on the third attempt
-    yield ('restart', 'restart.html', ())
 
   @classmethod
   def _CreateExpectations(cls):
@@ -81,26 +75,15 @@ def StartBrowser(cls):
     super(SimpleIntegrationUnittest, cls).StartBrowser()
     cls._num_browser_starts += 1
 
-  @classmethod
-  def StopBrowser(cls):
-    global _GLOBAL_RESTART_CRASH
-    if _GLOBAL_RESTART_CRASH:
-      if cls._num_restart_failures < 2:
-        cls._num_restart_failures += 1
-        raise Exception
-      else:
-        _GLOBAL_RESTART_CRASH = False
-
-    super(SimpleIntegrationUnittest, cls).StopBrowser()
-
-
   def RunActualGpuTest(self, file_path, *args):
     if file_path == 'failure.html':
       self.fail('Expected failure')
     elif file_path == 'restart.html':
-      global _GLOBAL_RESTART_CRASH
-      _GLOBAL_RESTART_CRASH = True
-      self._RestartBrowser("testing restart on failure")
+      try:
+        # This will fail because the browser is already started
+        self.StartBrowser()
+      finally:
+        self.StopBrowser()
     elif file_path == 'flaky.html':
       if self.__class__._num_flaky_runs_to_fail > 0:
         self.__class__._num_flaky_runs_to_fail -= 1
@@ -109,9 +92,73 @@ def RunActualGpuTest(self, file_path, *args):
       raise Exception('Expected exception')
 
 
+# TODO(eyaich@): add the actual unittest for start-up retrying logic.
+class BrowserStartFailureIntegrationUnittest(
+  gpu_integration_test.GpuIntegrationTest):
+  # Must be class-scoped since instances aren't reused across runs.
+  _num_restart_failures = 0
+
+  @classmethod
+  def setUpClass(cls):
+    finder_options = fakes.CreateBrowserFinderOptions()
+    finder_options.browser_options.platform = fakes.FakeLinuxPlatform()
+    finder_options.output_formats = ['none']
+    finder_options.suppress_gtest_report = True
+    finder_options.output_dir = None
+    finder_options .upload_bucket = 'public'
+    finder_options .upload_results = False
+    cls._finder_options = finder_options
+    cls.platform = None
+    cls.browser = None
+    cls.SetBrowserOptions(cls._finder_options)
+    cls.StartBrowser()
+
+  @classmethod
+  def _CreateExpectations(cls):
+    expectations = gpu_test_expectations.GpuTestExpectations()
+    expectations.Fail('expected_failure')
+    expectations.Flaky('expected_flaky', max_num_retries=3)
+    expectations.Skip('expected_skip')
+    return expectations
+
+  @classmethod
+  def Name(cls):
+    return 'browser_start_failure_integration_unittest'
+
+  @classmethod
+  def GenerateGpuTests(cls, options):
+    # This test causes the browser to try and restart the browser 3 times.
+    yield ('restart', 'restart.html', ())
+
+  def RunActualGpuTest(self, file_path, *args):
+    if file_path == 'restart.html':
+      try:
+        # This will fail because the browser is already started
+        self.StartBrowser()
+      finally:
+        self.StopBrowser()
+
+
 class GpuIntegrationTestUnittest(unittest.TestCase):
   @mock.patch('telemetry.internal.util.binary_manager.InitDependencyManager')
   def testSimpleIntegrationUnittest(self, mockInitDependencyManager):
+    self._RunIntegrationTest(
+      'simple_integration_unittest', [
+          'expected_failure',
+          'setup',
+          'unexpected_error',
+          'unexpected_failure'], ['expected_flaky'])
+    # It might be nice to be more precise about the order of operations
+    # with these browser restarts, but this is at least a start.
+    self.assertEquals(SimpleIntegrationUnittest._num_browser_starts, 6)
+
+  @mock.patch('telemetry.internal.util.binary_manager.InitDependencyManager')
+  def testIntegrationUnittestWithBrowserFailure(
+      self, mockInitDependencyManager):
+    self._RunIntegrationTest(
+      'browser_start_failure_integration_unittest', ['restart'], [])
+
+  def _RunIntegrationTest(self, test_name, failures, successes):
     options = browser_test_runner.TestRunOptions()
     # Suppress printing out information for passing tests.
     options.verbosity = 0
@@ -122,22 +169,13 @@ def testSimpleIntegrationUnittest(self, mockInitDependencyManager):
     try:
       browser_test_runner.Run(
           config, options,
-          ['simple_integration_unittest',
+          [test_name,
            '--write-abbreviated-json-results-to=%s' % temp_file_name])
       with open(temp_file_name) as f:
         test_result = json.load(f)
-      self.assertEquals(test_result['failures'], [
-          'expected_failure',
-          'setup',
-          'unexpected_error',
-          'unexpected_failure'])
-      self.assertEquals(test_result['successes'], [
-          'expected_flaky', 'restart'])
+      self.assertEquals(test_result['failures'], failures)
+      self.assertEquals(test_result['successes'], successes)
       self.assertEquals(test_result['valid'], True)
-      # It might be nice to be more precise about the order of operations
-      # with these browser restarts, but this is at least a start.
-      self.assertEquals(SimpleIntegrationUnittest._num_browser_starts, 7)
-      # Assert that we restarted the browser 2 times due to failure in restart
-      self.assertEquals(SimpleIntegrationUnittest._num_restart_failures, 2)
+
     finally:
       os.remove(temp_file_name)