[jobs] testing: reduce test flakiness

fixes GoogleCloudPlatform#2945 fixes GoogleCloudPlatform#2818 There were 8 tests which has `time.sleep(10)` for waiting data propagation or whatever. I chose to use `@eventually_consistent.call` instead of sleep in those tests. Bonus point is now the tests are significantly faster because it doesn't always wait for 10 seconds. The downside is it will take a long time on failure. We may want to specify the retry count on the eventualy consistent decorator once the following issue is fixed: GoogleCloudPlatform/python-repo-tools#25
tmatsuo · Apr 3, 2020 · 1d92840 · 1d92840
1 parent f8a1d3f
commit 1d92840
Show file tree

Hide file tree

Showing 17 changed files with 318 additions and 137 deletions.
diff --git a/jobs/v3/api_client/auto_complete_sample.py b/jobs/v3/api_client/auto_complete_sample.py
@@ -50,10 +50,9 @@ def auto_complete_default(client_service, query, company_name):
 # [END auto_complete_default]
 
 
-def run_sample():
+def set_up():
     import base_company_sample
     import base_job_sample
-
     company_to_be_created = base_company_sample.generate_company()
     company_created = base_company_sample.create_company(
         client_service, company_to_be_created)
@@ -64,16 +63,25 @@ def run_sample():
     job_to_be_created.update({'title': 'Software engineer'})
     job_name = base_job_sample.create_job(client_service,
                                           job_to_be_created).get('name')
+    return company_name, job_name
 
-    # Wait several seconds for post processing
-    time.sleep(10)
-    auto_complete_default(client_service, 'goo', company_name)
-    auto_complete_default(client_service, 'sof', company_name)
-    job_title_auto_complete(client_service, 'sof', company_name)
 
+def tear_down(company_name, job_name):
+    import base_company_sample
+    import base_job_sample
     base_job_sample.delete_job(client_service, job_name)
     base_company_sample.delete_company(client_service, company_name)
 
 
+def run_sample(company_name):
+    auto_complete_default(client_service, 'goo', company_name)
+    auto_complete_default(client_service, 'sof', company_name)
+    job_title_auto_complete(client_service, 'sof', company_name)
+
+
 if __name__ == '__main__':
-    run_sample()
+    company_name, job_name = set_up()
+    # Wait several seconds for post processing
+    time.sleep(10)
+    run_sample(company_name)
+    tear_down(company_name, job_name)
diff --git a/jobs/v3/api_client/auto_complete_sample_test.py b/jobs/v3/api_client/auto_complete_sample_test.py
@@ -12,19 +12,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pytest
+import re
 
-def test_auto_complete_sample(capsys):
-    import auto_complete_sample
-    import re
+from gcp_devrel.testing import eventually_consistent
 
-    auto_complete_sample.run_sample()
-    out, _ = capsys.readouterr()
-    expected = (
-        '.*completionResults.*'
-        'suggestion.*Google.*type.*COMPANY_NAME.*\n'
-        '.*completionResults.*'
-        'suggestion.*Software Engineer.*type.*JOB_TITLE.*\n'
-        '.*completionResults.*'
-        'suggestion.*Software Engineer.*type.*JOB_TITLE.*\n'
-    )
-    assert re.search(expected, out)
+import auto_complete_sample
+
+
+@pytest.fixture(scope="module")
+def company_name():
+    company_name, job_name = auto_complete_sample.set_up()
+    yield company_name
+    auto_complete_sample.tear_down(company_name, job_name)
+
+
+def test_auto_complete_sample(company_name, capsys):
+    @eventually_consistent.call
+    def _():
+        auto_complete_sample.run_sample(company_name)
+        out, _ = capsys.readouterr()
+        expected = (
+            '.*completionResults.*'
+            'suggestion.*Google.*type.*COMPANY_NAME.*\n'
+            '.*completionResults.*'
+            'suggestion.*Software Engineer.*type.*JOB_TITLE.*\n'
+            '.*completionResults.*'
+            'suggestion.*Software Engineer.*type.*JOB_TITLE.*\n'
+        )
+        assert re.search(expected, out)
diff --git a/jobs/v3/api_client/commute_search_sample.py b/jobs/v3/api_client/commute_search_sample.py
@@ -19,7 +19,6 @@
 import time
 
 from googleapiclient.discovery import build
-
 client_service = build('jobs', 'v3')
 parent = 'projects/' + os.environ['GOOGLE_CLOUD_PROJECT']
 # [END instantiate]
@@ -54,7 +53,7 @@ def commute_search(client_service, company_name):
 # [END commute_search]
 
 
-def run_sample():
+def set_up():
     import base_company_sample
     import base_job_sample
 
@@ -70,14 +69,24 @@ def run_sample():
     })
     job_name = base_job_sample.create_job(client_service,
                                           job_to_be_created).get('name')
+    return company_name, job_name
 
-    # Wait several seconds for post processing
-    time.sleep(10)
-    commute_search(client_service, company_name)
+
+def tear_down(company_name, job_name):
+    import base_company_sample
+    import base_job_sample
 
     base_job_sample.delete_job(client_service, job_name)
     base_company_sample.delete_company(client_service, company_name)
 
 
+def run_sample(company_name):
+    commute_search(client_service, company_name)
+
+
 if __name__ == '__main__':
-    run_sample()
+    company_name, job_name = set_up()
+    # Wait several seconds for post processing
+    time.sleep(10)
+    run_sample(company_name)
+    tear_down(company_name, job_name)
diff --git a/jobs/v3/api_client/commute_search_sample_test.py b/jobs/v3/api_client/commute_search_sample_test.py
@@ -12,12 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pytest
+import re
 
-def test_commute_search_sample(capsys):
-    import commute_search_sample
-    import re
+from gcp_devrel.testing import eventually_consistent
 
-    commute_search_sample.run_sample()
-    out, _ = capsys.readouterr()
-    expected = ('.*matchingJobs.*1600 Amphitheatre Pkwy.*')
-    assert re.search(expected, out)
+import commute_search_sample
+
+
+@pytest.fixture(scope="module")
+def company_name():
+    company_name, job_name = commute_search_sample.set_up()
+    yield company_name
+    commute_search_sample.tear_down(company_name, job_name)
+
+
+def test_commute_search_sample(company_name, capsys):
+    @eventually_consistent.call
+    def _():
+        commute_search_sample.run_sample(company_name)
+        out, _ = capsys.readouterr()
+        expected = ('.*matchingJobs.*1600 Amphitheatre Pkwy.*')
+        assert re.search(expected, out)
diff --git a/jobs/v3/api_client/custom_attribute_sample.py b/jobs/v3/api_client/custom_attribute_sample.py
@@ -133,7 +133,7 @@ def custom_attribute_filter_multi_attributes(client_service):
 # [END custom_attribute_filter_multi_attributes]
 
 
-def run_sample():
+def set_up():
     import base_company_sample
     import base_job_sample
 
@@ -145,16 +145,24 @@ def run_sample():
     job_to_be_created = generate_job_with_custom_attributes(company_name)
     job_name = base_job_sample.create_job(client_service,
                                           job_to_be_created).get('name')
+    return company_name, job_name
 
-    # Wait several seconds for post processing
-    time.sleep(10)
-    custom_attribute_filter_string_value(client_service)
-    custom_attribute_filter_long_value(client_service)
-    custom_attribute_filter_multi_attributes(client_service)
 
+def tear_down(company_name, job_name):
+    import base_company_sample
+    import base_job_sample
     base_job_sample.delete_job(client_service, job_name)
     base_company_sample.delete_company(client_service, company_name)
 
 
+def run_sample():
+    custom_attribute_filter_string_value(client_service)
+    custom_attribute_filter_long_value(client_service)
+    custom_attribute_filter_multi_attributes(client_service)
+
+
 if __name__ == '__main__':
-    run_sample()
+    company_name, job_name = set_up()
+    # Wait several seconds for post processing
+    time.sleep(10)
+    run_sample(company_name, job_name)
diff --git a/jobs/v3/api_client/custom_attribute_sample_test.py b/jobs/v3/api_client/custom_attribute_sample_test.py
@@ -12,15 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pytest
+import re
 
-def test_custom_attribute_sample(capsys):
-    import custom_attribute_sample
-    import re
+from gcp_devrel.testing import eventually_consistent
 
-    custom_attribute_sample.run_sample()
-    out, _ = capsys.readouterr()
-    expected = ('.*Job created:.*job_with_custom_attributes.*\n'
-                '.*matchingJobs.*job_with_custom_attributes.*\n'
-                '.*matchingJobs.*job_with_custom_attributes.*\n'
-                '.*matchingJobs.*job_with_custom_attributes.*\n')
-    assert re.search(expected, out, re.DOTALL)
+import custom_attribute_sample
+
+
+@pytest.fixture(scope="module")
+def create_data():
+    company_name, job_name = custom_attribute_sample.set_up()
+    yield
+    custom_attribute_sample.tear_down(company_name, job_name)
+
+
+def test_custom_attribute_sample(create_data, capsys):
+    @eventually_consistent.call
+    def _():
+        custom_attribute_sample.run_sample()
+        out, _ = capsys.readouterr()
+        expected = ('.*matchingJobs.*job_with_custom_attributes.*\n'
+                    '.*matchingJobs.*job_with_custom_attributes.*\n'
+                    '.*matchingJobs.*job_with_custom_attributes.*\n')
+        assert re.search(expected, out, re.DOTALL)
diff --git a/jobs/v3/api_client/email_alert_search_sample.py b/jobs/v3/api_client/email_alert_search_sample.py
@@ -44,7 +44,7 @@ def search_for_alerts(client_service, company_name):
 # [END search_for_alerts]
 
 
-def run_sample():
+def set_up():
     import base_company_sample
     import base_job_sample
 
@@ -58,13 +58,23 @@ def run_sample():
     job_name = base_job_sample.create_job(client_service,
                                           job_to_be_created).get('name')
 
-    # Wait several seconds for post processing
-    time.sleep(10)
-    search_for_alerts(client_service, company_name)
+    return company_name, job_name
 
+
+def tear_down(company_name, job_name):
+    import base_company_sample
+    import base_job_sample
     base_job_sample.delete_job(client_service, job_name)
     base_company_sample.delete_company(client_service, company_name)
 
 
+def run_sample(company_name):
+    search_for_alerts(client_service, company_name)
+
+
 if __name__ == '__main__':
-    run_sample()
+    company_name, job_name = set_up()
+    # Wait several seconds for post processing
+    time.sleep(10)
+    run_sample(company_name)
+    tear_down(company_name, job_name)
diff --git a/jobs/v3/api_client/email_alert_search_sample_test.py b/jobs/v3/api_client/email_alert_search_sample_test.py
@@ -12,12 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pytest
+import re
 
-def test_email_alert_search_sample(capsys):
-    import email_alert_search_sample
-    import re
+from gcp_devrel.testing import eventually_consistent
 
-    email_alert_search_sample.run_sample()
-    out, _ = capsys.readouterr()
-    expected = ('.*matchingJobs.*')
-    assert re.search(expected, out)
+import email_alert_search_sample
+
+
+@pytest.fixture(scope="module")
+def company_name():
+    company_name, job_name = email_alert_search_sample.set_up()
+    yield company_name
+    email_alert_search_sample.tear_down(company_name, job_name)
+
+
+def test_email_alert_search_sample(company_name, capsys):
+    @eventually_consistent.call
+    def _():
+        email_alert_search_sample.run_sample(company_name)
+        out, _ = capsys.readouterr()
+        expected = ('.*matchingJobs.*')
+        assert re.search(expected, out)
diff --git a/jobs/v3/api_client/featured_job_search_sample.py b/jobs/v3/api_client/featured_job_search_sample.py
@@ -74,7 +74,7 @@ def search_featured_job(client_service, company_name):
 # [END search_featured_job]
 
 
-def run_sample():
+def set_up():
     import base_company_sample
     import base_job_sample
 
@@ -86,14 +86,24 @@ def run_sample():
     job_to_be_created = generate_featured_job(company_name)
     job_name = base_job_sample.create_job(client_service,
                                           job_to_be_created).get('name')
+    return company_name, job_name
 
-    # Wait several seconds for post processing
-    time.sleep(10)
-    search_featured_job(client_service, company_name)
+
+def tear_down(company_name, job_name):
+    import base_company_sample
+    import base_job_sample
 
     base_job_sample.delete_job(client_service, job_name)
     base_company_sample.delete_company(client_service, company_name)
 
 
+def run_sample(company_name):
+    search_featured_job(client_service, company_name)
+
+
 if __name__ == '__main__':
-    run_sample()
+    company_name, job_name = set_up()
+    # Wait several seconds for post processing
+    time.sleep(10)
+    run_sample(company_name)
+    tear_down(company_name, job_name)
diff --git a/jobs/v3/api_client/featured_job_search_sample_test.py b/jobs/v3/api_client/featured_job_search_sample_test.py
@@ -12,12 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pytest
+import re
 
-def test_featured_job_search_sample(capsys):
-    import featured_job_search_sample
-    import re
+from gcp_devrel.testing import eventually_consistent
 
-    featured_job_search_sample.run_sample()
-    out, _ = capsys.readouterr()
-    expected = ('.*matchingJobs.*')
-    assert re.search(expected, out)
+import featured_job_search_sample
+
+
+@pytest.fixture(scope="module")
+def company_name():
+    company_name, job_name = featured_job_search_sample.set_up()
+    yield company_name
+    featured_job_search_sample.tear_down(company_name, job_name)
+
+
+def test_featured_job_search_sample(company_name, capsys):
+    @eventually_consistent.call
+    def _():
+        featured_job_search_sample.run_sample(company_name)
+        out, _ = capsys.readouterr()
+        expected = ('.*matchingJobs.*')
+        assert re.search(expected, out)