diff --git a/datalabeling/create_annotation_spec_set_test.py b/datalabeling/create_annotation_spec_set_test.py index 6ee2d31e71df..bcbfb7cb339b 100644 --- a/datalabeling/create_annotation_spec_set_test.py +++ b/datalabeling/create_annotation_spec_set_test.py @@ -39,7 +39,8 @@ def cleaner(): def test_create_annotation_spec_set(cleaner, capsys): - @backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=60) + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) def run_sample(): return create_annotation_spec_set.create_annotation_spec_set(PROJECT_ID) diff --git a/datalabeling/create_instruction_test.py b/datalabeling/create_instruction_test.py index 1d2930b2f412..183704b2bd7d 100644 --- a/datalabeling/create_instruction_test.py +++ b/datalabeling/create_instruction_test.py @@ -16,35 +16,39 @@ import os -from google.api_core.client_options import ClientOptions -from google.cloud import datalabeling_v1beta1 as datalabeling +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest import create_instruction +import testing_lib + PROJECT_ID = os.getenv('GCLOUD_PROJECT') INSTRUCTION_GCS_URI = ('gs://cloud-samples-data/datalabeling' '/instruction/test.pdf') -@pytest.mark.flaky(max_runs=3) -def test_create_instruction(capsys): - result = create_instruction.create_instruction( - PROJECT_ID, - 'IMAGE', - INSTRUCTION_GCS_URI - ) - out, _ = capsys.readouterr() - assert 'The instruction resource name: ' in out +@pytest.fixture(scope='module') +def cleaner(): + resource_names = [] + + yield resource_names + + for resource_name in resource_names: + testing_lib.delete_instruction(resource_name) - # Delete the created instruction. - instruction_name = result.name - client = datalabeling.DataLabelingServiceClient() - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) +def test_create_instruction(cleaner, capsys): - client.delete_instruction(instruction_name) + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + return create_instruction.create_instruction( + PROJECT_ID, 'IMAGE', INSTRUCTION_GCS_URI) + + instruction = run_sample() + cleaner.append(instruction.name) + + out, _ = capsys.readouterr() + assert 'The instruction resource name: ' in out diff --git a/datalabeling/import_data_test.py b/datalabeling/import_data_test.py index f3a63c904601..009349b80676 100644 --- a/datalabeling/import_data_test.py +++ b/datalabeling/import_data_test.py @@ -16,10 +16,14 @@ import os +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest import import_data import manage_dataset +import testing_lib + PROJECT_ID = os.getenv('GCLOUD_PROJECT') INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/image/image_dataset.csv' @@ -27,17 +31,33 @@ @pytest.fixture(scope='function') def dataset(): - # create a temporary dataset - dataset = manage_dataset.create_dataset(PROJECT_ID) + + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def create_dataset(): + # create a temporary dataset + return manage_dataset.create_dataset(PROJECT_ID) + + dataset = create_dataset() yield dataset - # tear down - manage_dataset.delete_dataset(dataset.name) + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def delete_dataset(): + # tear down + manage_dataset.delete_dataset(dataset.name) + + delete_dataset() -@pytest.mark.flaky(max_runs=3) def test_import_data(capsys, dataset): - import_data.import_data(dataset.name, 'IMAGE', INPUT_GCS_URI) + + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + import_data.import_data(dataset.name, 'IMAGE', INPUT_GCS_URI) + + run_sample() out, _ = capsys.readouterr() assert 'Dataset resource name: ' in out diff --git a/datalabeling/label_image_test.py b/datalabeling/label_image_test.py index b7fb13b68099..6fb0e66495be 100644 --- a/datalabeling/label_image_test.py +++ b/datalabeling/label_image_test.py @@ -16,101 +16,77 @@ import os -from google.api_core.client_options import ClientOptions -from google.cloud import datalabeling_v1beta1 as datalabeling +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest -import create_annotation_spec_set -import create_instruction -import import_data import label_image -import manage_dataset +import testing_lib + PROJECT_ID = os.getenv('GCLOUD_PROJECT') INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/image/image_dataset.csv' +INSTRUCTION_GCS_URI = ('gs://cloud-samples-data/datalabeling' + '/instruction/test.pdf') @pytest.fixture(scope='function') def dataset(): # create a temporary dataset - dataset = manage_dataset.create_dataset(PROJECT_ID) - - # import some data to it - import_data.import_data(dataset.name, 'IMAGE', INPUT_GCS_URI) + dataset = testing_lib.create_dataset(PROJECT_ID) yield dataset # tear down - manage_dataset.delete_dataset(dataset.name) + testing_lib.delete_dataset(dataset.name) @pytest.fixture(scope='function') def annotation_spec_set(): # create a temporary annotation_spec_set - response = create_annotation_spec_set.create_annotation_spec_set( - PROJECT_ID) + response = testing_lib.create_annotation_spec_set(PROJECT_ID) yield response - # tear down - client = datalabeling.DataLabelingServiceClient() - - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - - client.delete_annotation_spec_set(response.name) + testing_lib.delete_annotation_spec_set(response.name) @pytest.fixture(scope='function') def instruction(): # create a temporary instruction - instruction = create_instruction.create_instruction( - PROJECT_ID, 'IMAGE', - 'gs://cloud-samples-data/datalabeling/instruction/test.pdf') + instruction = testing_lib.create_instruction( + PROJECT_ID, 'IMAGE', INSTRUCTION_GCS_URI) yield instruction # tear down - client = datalabeling.DataLabelingServiceClient() + testing_lib.delete_instruction(instruction.name) + - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) +@pytest.fixture(scope='module') +def cleaner(): + resource_names = [] - client.delete_instruction(instruction.name) + yield resource_names + + for resource_name in resource_names: + testing_lib.cancel_operation(resource_name) # Passing in dataset as the last argument in test_label_image since it needs # to be deleted before the annotation_spec_set can be deleted. -@pytest.mark.flaky(max_runs=3) -def test_label_image(capsys, annotation_spec_set, instruction, dataset): - - # Start labeling. - response = label_image.label_image( - dataset.name, - instruction.name, - annotation_spec_set.name - ) - out, _ = capsys.readouterr() - assert 'Label_image operation name: ' in out - operation_name = response.operation.name +def test_label_image( + capsys, annotation_spec_set, instruction, dataset, cleaner): - # Cancels the labeling operation. - response.cancel() - assert response.cancelled() is True + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + # Start labeling. + return label_image.label_image( + dataset.name, instruction.name, annotation_spec_set.name) - client = datalabeling.DataLabelingServiceClient() + response = run_sample() + cleaner.append(response.operation.name) - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - - client.transport._operations_client.cancel_operation( - operation_name) + out, _ = capsys.readouterr() + assert 'Label_image operation name: ' in out diff --git a/datalabeling/label_text_test.py b/datalabeling/label_text_test.py index 5b2cc51bc934..1c0c28601d04 100644 --- a/datalabeling/label_text_test.py +++ b/datalabeling/label_text_test.py @@ -16,101 +16,81 @@ import os -from google.api_core.client_options import ClientOptions -from google.cloud import datalabeling_v1beta1 as datalabeling +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest -import create_annotation_spec_set -import create_instruction -import import_data import label_text -import manage_dataset +import testing_lib PROJECT_ID = os.getenv('GCLOUD_PROJECT') INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/text/input.csv' +INSTRUCTION_GCS_URI = ('gs://cloud-samples-data/datalabeling' + '/instruction/test.pdf') @pytest.fixture(scope='function') def dataset(): # create a temporary dataset - dataset = manage_dataset.create_dataset(PROJECT_ID) + dataset = testing_lib.create_dataset(PROJECT_ID) - # import some data to it - import_data.import_data(dataset.name, 'TEXT', INPUT_GCS_URI) + testing_lib.import_data(dataset.name, 'TEXT', INPUT_GCS_URI) yield dataset # tear down - manage_dataset.delete_dataset(dataset.name) + testing_lib.delete_dataset(dataset.name) @pytest.fixture(scope='function') def annotation_spec_set(): # create a temporary annotation_spec_set - response = create_annotation_spec_set.create_annotation_spec_set( - PROJECT_ID) + response = testing_lib.create_annotation_spec_set(PROJECT_ID) yield response - # tear down - client = datalabeling.DataLabelingServiceClient() - - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - - client.delete_annotation_spec_set(response.name) + testing_lib.delete_annotation_spec_set(response.name) @pytest.fixture(scope='function') def instruction(): # create a temporary instruction - instruction = create_instruction.create_instruction( - PROJECT_ID, 'TEXT', - 'gs://cloud-samples-data/datalabeling/instruction/test.pdf') + instruction = testing_lib.create_instruction( + PROJECT_ID, 'IMAGE', INSTRUCTION_GCS_URI) yield instruction # tear down - client = datalabeling.DataLabelingServiceClient() + testing_lib.delete_instruction(instruction.name) - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - client.delete_instruction(instruction.name) +@pytest.fixture(scope='module') +def cleaner(): + resource_names = [] + + yield resource_names + + for resource_name in resource_names: + testing_lib.cancel_operation(resource_name) # Passing in dataset as the last argument in test_label_image since it needs # to be deleted before the annotation_spec_set can be deleted. -@pytest.mark.flaky(max_runs=3) -def test_label_text(capsys, annotation_spec_set, instruction, dataset): - - # Start labeling. - response = label_text.label_text( - dataset.name, - instruction.name, - annotation_spec_set.name - ) +def test_label_text(capsys, annotation_spec_set, instruction, dataset, cleaner): + + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + # Start labeling. + return label_text.label_text( + dataset.name, instruction.name, annotation_spec_set.name) + + response = run_sample() + cleaner.append(response.operation.name) + out, _ = capsys.readouterr() assert 'Label_text operation name: ' in out - operation_name = response.operation.name # Cancels the labeling operation. response.cancel() assert response.cancelled() is True - - client = datalabeling.DataLabelingServiceClient() - - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - - client.transport._operations_client.cancel_operation( - operation_name) diff --git a/datalabeling/label_video_test.py b/datalabeling/label_video_test.py index 2c8b79de7565..b98849cca4be 100644 --- a/datalabeling/label_video_test.py +++ b/datalabeling/label_video_test.py @@ -16,101 +16,82 @@ import os -from google.api_core.client_options import ClientOptions -from google.cloud import datalabeling_v1beta1 as datalabeling +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest -import create_annotation_spec_set -import create_instruction -import import_data import label_video -import manage_dataset +import testing_lib PROJECT_ID = os.getenv('GCLOUD_PROJECT') INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/videos/video_dataset.csv' +INSTRUCTION_GCS_URI = ('gs://cloud-samples-data/datalabeling' + '/instruction/test.pdf') @pytest.fixture(scope='function') def dataset(): # create a temporary dataset - dataset = manage_dataset.create_dataset(PROJECT_ID) + dataset = testing_lib.create_dataset(PROJECT_ID) - # import some data to it - import_data.import_data(dataset.name, 'VIDEO', INPUT_GCS_URI) + testing_lib.import_data(dataset.name, 'VIDEO', INPUT_GCS_URI) yield dataset # tear down - manage_dataset.delete_dataset(dataset.name) + testing_lib.delete_dataset(dataset.name) @pytest.fixture(scope='function') def annotation_spec_set(): # create a temporary annotation_spec_set - response = create_annotation_spec_set.create_annotation_spec_set( - PROJECT_ID) + response = testing_lib.create_annotation_spec_set(PROJECT_ID) yield response - # tear down - client = datalabeling.DataLabelingServiceClient() - - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - - client.delete_annotation_spec_set(response.name) + testing_lib.delete_annotation_spec_set(response.name) @pytest.fixture(scope='function') def instruction(): # create a temporary instruction - instruction = create_instruction.create_instruction( - PROJECT_ID, 'VIDEO', - 'gs://cloud-samples-data/datalabeling/instruction/test.pdf') + instruction = testing_lib.create_instruction( + PROJECT_ID, 'VIDEO', INSTRUCTION_GCS_URI) yield instruction # tear down - client = datalabeling.DataLabelingServiceClient() + testing_lib.delete_instruction(instruction.name) - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - client.delete_instruction(instruction.name) +@pytest.fixture(scope='module') +def cleaner(): + resource_names = [] + + yield resource_names + + for resource_name in resource_names: + testing_lib.cancel_operation(resource_name) # Passing in dataset as the last argument in test_label_image since it needs # to be deleted before the annotation_spec_set can be deleted. -@pytest.mark.flaky(max_runs=3) -def test_label_video(capsys, annotation_spec_set, instruction, dataset): - - # Start labeling. - response = label_video.label_video( - dataset.name, - instruction.name, - annotation_spec_set.name - ) +def test_label_video( + capsys, annotation_spec_set, instruction, dataset, cleaner): + + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + # Start labeling. + return label_video.label_video( + dataset.name, instruction.name, annotation_spec_set.name) + + response = run_sample() + cleaner.append(response) + out, _ = capsys.readouterr() assert 'Label_video operation name: ' in out - operation_name = response.operation.name # Cancels the labeling operation. response.cancel() assert response.cancelled() is True - - client = datalabeling.DataLabelingServiceClient() - - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - - client.transport._operations_client.cancel_operation( - operation_name) diff --git a/datalabeling/manage_dataset_test.py b/datalabeling/manage_dataset_test.py index 6b8fd3828a28..eb8824c3faa0 100644 --- a/datalabeling/manage_dataset_test.py +++ b/datalabeling/manage_dataset_test.py @@ -16,53 +16,81 @@ import os +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest import manage_dataset +import testing_lib + PROJECT_ID = os.getenv("GCLOUD_PROJECT") -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def dataset(): # create a temporary dataset - dataset = manage_dataset.create_dataset(PROJECT_ID) + dataset = testing_lib.create_dataset(PROJECT_ID) yield dataset # tear down - manage_dataset.delete_dataset(dataset.name) + testing_lib.delete_dataset(dataset.name) + + +@pytest.fixture(scope='module') +def cleaner(): + resource_names = [] + + yield resource_names + + for resource_name in resource_names: + testing_lib.delete_dataset(resource_name) + +def test_create_dataset(cleaner, capsys): + + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + return manage_dataset.create_dataset(PROJECT_ID) + + response = run_sample() + cleaner.append(response.name) -@pytest.mark.flaky(max_runs=3) -def test_create_dataset(capsys): - response = manage_dataset.create_dataset(PROJECT_ID) out, _ = capsys.readouterr() assert "The dataset resource name:" in out - # clean up - manage_dataset.delete_dataset(response.name) - -@pytest.mark.flaky(max_runs=3) def test_list_dataset(capsys, dataset): - manage_dataset.list_datasets(PROJECT_ID) + + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + manage_dataset.list_datasets(PROJECT_ID) + + run_sample() out, _ = capsys.readouterr() assert dataset.name in out -@pytest.mark.flaky(max_runs=3) def test_get_dataset(capsys, dataset): - manage_dataset.get_dataset(dataset.name) + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + manage_dataset.get_dataset(dataset.name) + + run_sample() out, _ = capsys.readouterr() assert "The dataset resource name:" in out -@pytest.mark.flaky(max_runs=3) -def test_delete_dataset(capsys): - # Creates a dataset. - response = manage_dataset.create_dataset(PROJECT_ID) +def test_delete_dataset(capsys, dataset): + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + manage_dataset.delete_dataset(dataset.name) - manage_dataset.delete_dataset(response.name) + run_sample() out, _ = capsys.readouterr() assert "Dataset deleted." in out diff --git a/datalabeling/requirements-test.txt b/datalabeling/requirements-test.txt index 40857e816e6b..8855f3cf1f88 100644 --- a/datalabeling/requirements-test.txt +++ b/datalabeling/requirements-test.txt @@ -1,3 +1,2 @@ backoff==1.10.0 pytest==5.3.2 -flaky==3.6.1 diff --git a/datalabeling/testing_lib.py b/datalabeling/testing_lib.py index 8f7628e4bc4b..e28da194a6a2 100644 --- a/datalabeling/testing_lib.py +++ b/datalabeling/testing_lib.py @@ -19,6 +19,13 @@ from google.api_core.exceptions import DeadlineExceeded from google.cloud import datalabeling_v1beta1 as datalabeling +import create_annotation_spec_set as annotation_spec_set_sample +import create_instruction as instruction_sample +import manage_dataset as dataset_sample +import import_data as import_sample + +RETRY_DEADLINE = 60 + def create_client(): # If provided, use a provided test endpoint - this will prevent tests on @@ -31,7 +38,44 @@ def create_client(): return client -@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=60) +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def create_dataset(project_id): + return dataset_sample.create_dataset(project_id) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def delete_dataset(name): + return dataset_sample.delete_dataset(name) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def create_annotation_spec_set(project_id): + return annotation_spec_set_sample.create_annotation_spec_set(project_id) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) def delete_annotation_spec_set(name): client = create_client() client.delete_annotation_spec_set(name) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def create_instruction(project_id, data_type, gcs_uri): + return instruction_sample.create_instruction(project_id, data_type, gcs_uri) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def delete_instruction(name): + client = create_client() + client.delete_instruction(name) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def cancel_operation(name): + client = create_client() + client.transport._operations_client.cancel_operation(name) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def import_data(dataset_name, data_type, gcs_uri): + import_sample.import_data(dataset_name, data_type, gcs_uri)