[vision] testing: retry upon errors (#3764)

fixes #3734

I only wrapped some of the tests. Potentially we can do it for
everything.

vision/cloud-client/detect/detect_test.py

@@ -15,6 +15,7 @@
 import os
 import uuid
 
+import backoff
 from google.cloud import storage
 import pytest
 
@@ -142,19 +143,40 @@ def test_detect_properties_uri(capsys):
     assert 'frac' in out
 
 
+def only_sample_error(e):
+    """A callback for giving up upon Exceptions.
+
+    Giving up upon any Exceptions other than the ones that sample code
+    throws at the end of the function.
+    """
+    return 'https://cloud.google.com/apis/design/errors' not in str(e)
+
+
 # Vision 1.1 tests
 def test_detect_web(capsys):
     file_name = os.path.join(
         os.path.dirname(__file__),
         'resources/landmark.jpg')
-    detect.detect_web(file_name)
+
+    @backoff.on_exception(
+        backoff.expo, Exception, max_time=60, giveup=only_sample_error)
+    def run_sample():
+        detect.detect_web(file_name)
+
+    run_sample()
     out, _ = capsys.readouterr()
     assert 'best guess label: palace of fine arts' in out.lower()
 
 
 def test_detect_web_uri(capsys):
     file_name = 'gs://{}/vision/landmark/pofa.jpg'.format(ASSET_BUCKET)
-    detect.detect_web_uri(file_name)
+
+    @backoff.on_exception(
+        backoff.expo, Exception, max_time=60, giveup=only_sample_error)
+    def run_sample():
+        detect.detect_web_uri(file_name)
+
+    run_sample()
     out, _ = capsys.readouterr()
     assert 'best guess label: palace of fine arts' in out.lower()
 
@@ -163,15 +185,27 @@ def test_detect_web_with_geo(capsys):
     file_name = os.path.join(
         os.path.dirname(__file__),
         'resources/city.jpg')
-    detect.web_entities_include_geo_results(file_name)
+
+    @backoff.on_exception(
+        backoff.expo, Exception, max_time=60, giveup=only_sample_error)
+    def run_sample():
+        detect.web_entities_include_geo_results(file_name)
+
+    run_sample()
     out, _ = capsys.readouterr()
     out = out.lower()
     assert 'description' in out
 
 
 def test_detect_web_with_geo_uri(capsys):
     file_name = 'gs://{}/vision/web/city.jpg'.format(ASSET_BUCKET)
-    detect.web_entities_include_geo_results_uri(file_name)
+
+    @backoff.on_exception(
+        backoff.expo, Exception, max_time=60, giveup=only_sample_error)
+    def run_sample():
+        detect.web_entities_include_geo_results_uri(file_name)
+
+    run_sample()
     out, _ = capsys.readouterr()
     out = out.lower()
     assert 'description' in out

vision/cloud-client/detect/requirements-test.txt

@@ -1,2 +1,3 @@
+backoff==1.10.0
 pytest==5.3.2
 flaky==3.6.1