[healthcare] fix: retry eventually consistent tests (#3434)

fixes #3432

Author: Takashi Matsuo

healthcare/api-client/v1/hl7v2/hl7v2_messages_test.py

@@ -17,8 +17,8 @@
 import sys
 import uuid
 
+import backoff
 from googleapiclient.errors import HttpError
-from retrying import retry
 
 # Add datasets for bootstrapping datasets for testing
 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "datasets"))  # noqa
@@ -36,18 +36,9 @@
 label_value = "TRUE"
 
 
-def retry_if_server_exception(exception):
-    return isinstance(exception, (HttpError))
-
-
 @pytest.fixture(scope="module")
 def test_dataset():
-    @retry(
-        wait_exponential_multiplier=1000,
-        wait_exponential_max=10000,
-        stop_max_attempt_number=10,
-        retry_on_exception=retry_if_server_exception,
-    )
+    @backoff.on_exception(backoff.expo, HttpError, max_time=60)
     def create():
         try:
             datasets.create_dataset(project_id, cloud_region, dataset_id)
@@ -65,12 +56,7 @@ def create():
     yield
 
     # Clean up
-    @retry(
-        wait_exponential_multiplier=1000,
-        wait_exponential_max=10000,
-        stop_max_attempt_number=10,
-        retry_on_exception=retry_if_server_exception,
-    )
+    @backoff.on_exception(backoff.expo, HttpError, max_time=60)
     def clean_up():
         try:
             datasets.delete_dataset(project_id, cloud_region, dataset_id)
@@ -86,12 +72,7 @@ def clean_up():
 
 @pytest.fixture(scope="module")
 def test_hl7v2_store():
-    @retry(
-        wait_exponential_multiplier=1000,
-        wait_exponential_max=10000,
-        stop_max_attempt_number=10,
-        retry_on_exception=retry_if_server_exception,
-    )
+    @backoff.on_exception(backoff.expo, HttpError, max_time=60)
     def create():
         try:
             hl7v2_stores.create_hl7v2_store(
@@ -115,12 +96,7 @@ def create():
     yield
 
     # Clean up
-    @retry(
-        wait_exponential_multiplier=1000,
-        wait_exponential_max=10000,
-        stop_max_attempt_number=10,
-        retry_on_exception=retry_if_server_exception,
-    )
+    @backoff.on_exception(backoff.expo, HttpError, max_time=60)
     def clean_up():
         try:
             hl7v2_stores.delete_hl7v2_store(
@@ -145,12 +121,20 @@ def test_CRUD_hl7v2_message(test_dataset, test_hl7v2_store, capsys):
         project_id, cloud_region, dataset_id, hl7v2_store_id, hl7v2_message_file
     )
 
-    hl7v2_messages_list = hl7v2_messages.list_hl7v2_messages(
-        project_id, cloud_region, dataset_id, hl7v2_store_id
-    )
+    @backoff.on_exception(backoff.expo, AssertionError, max_time=60)
+    def run_eventually_consistent_test():
+        hl7v2_messages_list = hl7v2_messages.list_hl7v2_messages(
+            project_id, cloud_region, dataset_id, hl7v2_store_id
+        )
+
+        assert len(hl7v2_messages_list) > 0
+        hl7v2_message_name = hl7v2_messages_list[0].get("name")
+        elms = hl7v2_message_name.split("/", 9)
+        assert len(elms) >= 10
+        hl7v2_message_id = elms[9]
+        return hl7v2_message_id
 
-    hl7v2_message_name = hl7v2_messages_list[0].get("name")
-    hl7v2_message_id = hl7v2_message_name.split("/", 9)[9]
+    hl7v2_message_id = run_eventually_consistent_test()
 
     hl7v2_messages.get_hl7v2_message(
         project_id, cloud_region, dataset_id, hl7v2_store_id, hl7v2_message_id
@@ -173,12 +157,20 @@ def test_ingest_hl7v2_message(test_dataset, test_hl7v2_store, capsys):
         project_id, cloud_region, dataset_id, hl7v2_store_id, hl7v2_message_file
     )
 
-    hl7v2_messages_list = hl7v2_messages.list_hl7v2_messages(
-        project_id, cloud_region, dataset_id, hl7v2_store_id
-    )
+    @backoff.on_exception(backoff.expo, AssertionError, max_time=60)
+    def run_eventually_consistent_test():
+        hl7v2_messages_list = hl7v2_messages.list_hl7v2_messages(
+            project_id, cloud_region, dataset_id, hl7v2_store_id
+        )
 
-    hl7v2_message_name = hl7v2_messages_list[0].get("name")
-    hl7v2_message_id = hl7v2_message_name.split("/", 9)[9]
+        assert len(hl7v2_messages_list) > 0
+        hl7v2_message_name = hl7v2_messages_list[0].get("name")
+        elms = hl7v2_message_name.split("/", 9)
+        assert len(elms) >= 10
+        hl7v2_message_id = elms[9]
+        return hl7v2_message_id
+
+    hl7v2_message_id = run_eventually_consistent_test()
 
     hl7v2_messages.get_hl7v2_message(
         project_id, cloud_region, dataset_id, hl7v2_store_id, hl7v2_message_id
@@ -201,13 +193,20 @@ def test_patch_hl7v2_message(test_dataset, test_hl7v2_store, capsys):
         project_id, cloud_region, dataset_id, hl7v2_store_id, hl7v2_message_file
     )
 
-    hl7v2_messages_list = hl7v2_messages.list_hl7v2_messages(
-        project_id, cloud_region, dataset_id, hl7v2_store_id
-    )
+    @backoff.on_exception(backoff.expo, AssertionError, max_time=60)
+    def run_eventually_consistent_test():
+        hl7v2_messages_list = hl7v2_messages.list_hl7v2_messages(
+            project_id, cloud_region, dataset_id, hl7v2_store_id
+        )
+
+        assert len(hl7v2_messages_list) > 0
+        hl7v2_message_name = hl7v2_messages_list[0].get("name")
+        elms = hl7v2_message_name.split("/", 9)
+        assert len(elms) >= 10
+        hl7v2_message_id = elms[9]
+        return hl7v2_message_id
 
-    assert len(hl7v2_messages_list) > 0
-    hl7v2_message_name = hl7v2_messages_list[0].get("name")
-    hl7v2_message_id = hl7v2_message_name.split("/", 9)[9]
+    hl7v2_message_id = run_eventually_consistent_test()
 
     hl7v2_messages.patch_hl7v2_message(
         project_id,

healthcare/api-client/v1/hl7v2/hl7v2_stores_test.py

@@ -17,8 +17,8 @@
 import sys
 import uuid
 
+import backoff
 from googleapiclient.errors import HttpError
-from retrying import retry
 
 # Add datasets for bootstrapping datasets for testing
 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "datasets"))  # noqa
@@ -38,12 +38,7 @@ def retry_if_server_exception(exception):
 
 @pytest.fixture(scope="module")
 def test_dataset():
-    @retry(
-        wait_exponential_multiplier=1000,
-        wait_exponential_max=10000,
-        stop_max_attempt_number=10,
-        retry_on_exception=retry_if_server_exception,
-    )
+    @backoff.on_exception(backoff.expo, HttpError, max_time=60)
     def create():
         try:
             datasets.create_dataset(project_id, cloud_region, dataset_id)
@@ -61,12 +56,7 @@ def create():
     yield
 
     # Clean up
-    @retry(
-        wait_exponential_multiplier=1000,
-        wait_exponential_max=10000,
-        stop_max_attempt_number=10,
-        retry_on_exception=retry_if_server_exception,
-    )
+    @backoff.on_exception(backoff.expo, HttpError, max_time=60)
     def clean_up():
         try:
             datasets.delete_dataset(project_id, cloud_region, dataset_id)
@@ -82,12 +72,7 @@ def clean_up():
 
 @pytest.fixture(scope="module")
 def test_hl7v2_store():
-    @retry(
-        wait_exponential_multiplier=1000,
-        wait_exponential_max=10000,
-        stop_max_attempt_number=10,
-        retry_on_exception=retry_if_server_exception,
-    )
+    @backoff.on_exception(backoff.expo, HttpError, max_time=60)
     def create():
         try:
             hl7v2_stores.create_hl7v2_store(
@@ -111,12 +96,7 @@ def create():
     yield
 
     # Clean up
-    @retry(
-        wait_exponential_multiplier=1000,
-        wait_exponential_max=10000,
-        stop_max_attempt_number=10,
-        retry_on_exception=retry_if_server_exception,
-    )
+    @backoff.on_exception(backoff.expo, HttpError, max_time=60)
     def clean_up():
         try:
             hl7v2_stores.delete_hl7v2_store(
@@ -141,12 +121,7 @@ def crud_hl7v2_store_id():
     yield hl7v2_store_id
 
     # Clean up
-    @retry(
-        wait_exponential_multiplier=1000,
-        wait_exponential_max=10000,
-        stop_max_attempt_number=10,
-        retry_on_exception=retry_if_server_exception,
-    )
+    @backoff.on_exception(backoff.expo, HttpError, max_time=60)
     def clean_up():
         try:
             hl7v2_stores.delete_hl7v2_store(

healthcare/api-client/v1/hl7v2/requirements-test.txt

@@ -1,2 +1,2 @@
 pytest==5.3.2
-retrying==1.3.3
+backoff==1.10.0