devil: Try safely handling 'connect for root closed' errors when rooting

We're seeing these errors during random `adb root` calls, which ends
up poisoning the rest of the test:
https://chromium-swarm.appspot.com/task?id=4ea8f76daeeb4d10

Hopefully with this extra wait-for-device call, we can let the rest of
the shard proceed safely with the retries of root it attempts.

Bug: chromium:1127009
Change-Id: I52f07394840fa1e5572f10841967e67487e97ce2
Reviewed-on: https://chromium-review.googlesource.com/c/catapult/+/2411339
Reviewed-by: John Budorick <jbudorick@chromium.org>
Commit-Queue: Ben Pastene <bpastene@chromium.org>

devil/devil/android/sdk/adb_wrapper.py

@@ -1069,7 +1069,16 @@ def Root(self, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES):
       timeout: (optional) Timeout per try in seconds.
       retries: (optional) Number of retries to attempt.
     """
-    output = self._RunDeviceAdbCmd(['root'], timeout, retries)
+    try:
+      output = self._RunDeviceAdbCmd(['root'], timeout, retries)
+    except device_errors.AdbCommandFailedError as e:
+      # For some devices, root can occasionally fail with this error and kick
+      # the device into adb 'offline' mode. Assuming this is transient, try
+      # waiting for the device to come back up before re-raising the exception
+      # and proceeding with any retries.
+      if 'unable to connect for root: closed' in e.output:
+        self.WaitForDevice()
+      raise
     if 'cannot' in output:
       raise device_errors.AdbCommandFailedError(
           ['root'], output, device_serial=self._device_serial)

devil/devil/android/sdk/adb_wrapper_test.py

@@ -68,3 +68,12 @@ def testWaitingForDevice(self, get_cmd_mock):
     get_cmd_mock.return_value = (1, '- waiting for device - ')
     self.assertRaises(device_errors.DeviceUnreachableError, self.adb.Shell,
                       '/bin/true')
+
+  @mock.patch('devil.utils.cmd_helper.GetCmdStatusAndOutputWithTimeout')
+  def testRootConnectionClosedFailure(self, get_cmd_mock):
+    get_cmd_mock.side_effect = [
+        (1, 'unable to connect for root: closed'),
+        (0, ''),
+    ]
+    self.assertRaises(device_errors.AdbCommandFailedError, self.adb.Root,
+                      retries=0)