[K/N][tests] Set timeout for gsutil and try repeating if it hits

This command (used for copying the iOS device test execution results
from the cloud) sometimes just hangs on certain CI agents.

This commit sets the timeout for it, and tries to repeat the execution
a couple of times to mitigate this. In a simple experiment, this
little trick proved to work.

^KT-72581 Fixed

native/executors/src/main/kotlin/org/jetbrains/kotlin/native/executors/ExecutorUtils.kt

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.native.executors
+
+import kotlin.time.Duration
+
+fun Executor.executeWithRepeatOnTimeout(request: ExecuteRequest, timeouts: List<Duration>): ExecuteResponse {
+    require(timeouts.isNotEmpty())
+    lateinit var response: ExecuteResponse
+
+    for (timeout in timeouts) {
+        response = execute(request.copying {
+            this.timeout = timeout
+        })
+        if (response.exitCode != null) {
+            return response
+        } else {
+            // It was killed by a timeout, let's repeat.
+        }
+    }
+
+    return response
+}

native/executors/src/main/kotlin/org/jetbrains/kotlin/native/executors/FirebaseCloudXCTestExecutor.kt

@@ -14,6 +14,8 @@ import java.nio.file.*
 import java.util.zip.ZipEntry
 import java.util.zip.ZipOutputStream
 import kotlin.io.path.*
+import kotlin.time.Duration.Companion.minutes
+import kotlin.time.Duration.Companion.seconds
 
 /**
  * Executes test cases on Firebase Test Lab for iOS arm64 devices.
@@ -114,11 +116,18 @@ class FirebaseCloudXCTestExecutor(
             ?: error("Unable to match URL against pattern, the input was: \"$firebaseStderrString\"")
 
         // Fetch results from the storage
-        hostExecutor.execute(
+        hostExecutor.executeWithRepeatOnTimeout(
             ExecuteRequest(
                 executableAbsolutePath = "gsutil",
                 workingDirectory = projectDir.toFile(),
                 args = mutableListOf("-m", "cp", "-r", "gs://${resultsBucketURL}/iphone*", ".")
+            ),
+            // This command sometimes just hangs on certain agents, see KT-72581.
+            // Let's try repeating.
+            timeouts = listOf(
+                10.seconds, // 3 seconds is always enough, so let's make it 10 just in case.
+                1.minutes, // Is 10 seconds not enough? Not typical, but I'll allow it.
+                2.minutes, // Ok, give it one last chance.
             )
         ).assertSuccess()
         val executionLog = projectDir.listDirectoryEntries("iphone*")