limit the max retry wait time

dlp/conftest.py

@@ -0,0 +1,20 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+
+# Used in risk_test.py to limit the maximum wait time before the flaky retries.
+def pytest_configure(config):
+    pytest.MAX_FLAKY_WAIT = 3600  # maximum of an hour

dlp/risk_test.py

@@ -172,14 +172,12 @@ def delay(err, *args):
     # slow which leads to the test failures. These situations tend to
     # get self healed in 20 minutes or so, so I'm trying this strategy.
     #
-    # The worst case execution time becomes longer, but I think it
-    # will give us higher success rate.
-    #
-    # There are ten tests, and each tests has 30 seconds wait time
-    # and retried 1 times, so the worst case latency for the waits are:
-    # 210 minutes (3 hours 30 minutes).
-    # This might cause time out on Kokoro.
-    time.sleep(60*20)
+    # There are 10 tests, so we don't want the retry delay happening
+    # for all the tests. When we exhaust the MAX_FLAKY_WAIT, we retry
+    # the test immediately.
+    wait_time = min(pytest.MAX_FLAKY_WAIT, 60*20)
+    pytest.MAX_FLAKY_WAIT -= wait_time
+    time.sleep(wait_time)
     return True