Skip to content

Commit

Permalink
[dlp] fix: remove gcp-devrel-py-tools
Browse files Browse the repository at this point in the history
fixes GoogleCloudPlatform#3375
fixes GoogleCloudPlatform#3416
fixes GoogleCloudPlatform#3417

* remove wrong usage of `eventually_consistent.call`
* only test if the operation has been started
* shorter timeout for polling
* correct use of `pytest.mark.flaky`
  • Loading branch information
Takashi Matsuo committed Apr 17, 2020
1 parent d66e3a7 commit ce10217
Show file tree
Hide file tree
Showing 5 changed files with 89 additions and 71 deletions.
3 changes: 3 additions & 0 deletions dlp/inspect_content.py
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,7 @@ def inspect_gcs_file(

operation = dlp.create_dlp_job(parent, inspect_job=inspect_job)
print("Inspection operation started: {}".format(operation.name))

# Create a Pub/Sub client and find the subscription. The subscription is
# expected to already be listening to the topic.
subscriber = google.cloud.pubsub.SubscriberClient()
Expand Down Expand Up @@ -636,6 +637,7 @@ def inspect_datastore(
}

operation = dlp.create_dlp_job(parent, inspect_job=inspect_job)
print("Inspection operation started: {}".format(operation.name))

# Create a Pub/Sub client and find the subscription. The subscription is
# expected to already be listening to the topic.
Expand Down Expand Up @@ -802,6 +804,7 @@ def inspect_bigquery(
}

operation = dlp.create_dlp_job(parent, inspect_job=inspect_job)
print("Inspection operation started: {}".format(operation.name))

# Create a Pub/Sub client and find the subscription. The subscription is
# expected to already be listening to the topic.
Expand Down
129 changes: 74 additions & 55 deletions dlp/inspect_content_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,18 @@
import os
import uuid

from gcp_devrel.testing import eventually_consistent
from gcp_devrel.testing.flaky import flaky
import google.api_core.exceptions
import google.cloud.bigquery
import google.cloud.datastore
import google.cloud.dlp_v2
import google.cloud.exceptions
import google.cloud.pubsub
import google.cloud.storage

import pytest

import inspect_content


UNIQUE_STRING = str(uuid.uuid4()).split("-")[0]

GCLOUD_PROJECT = os.getenv("GCLOUD_PROJECT")
Expand Down Expand Up @@ -95,7 +94,8 @@ def subscription_id(topic_id):
# Subscribes to a topic.
subscriber = google.cloud.pubsub.SubscriberClient()
topic_path = subscriber.topic_path(GCLOUD_PROJECT, topic_id)
subscription_path = subscriber.subscription_path(GCLOUD_PROJECT, SUBSCRIPTION_ID)
subscription_path = subscriber.subscription_path(
GCLOUD_PROJECT, SUBSCRIPTION_ID)
try:
subscriber.create_subscription(subscription_path, topic_path)
except google.api_core.exceptions.AlreadyExists:
Expand Down Expand Up @@ -297,21 +297,21 @@ def test_inspect_gcs_file(bucket, topic_id, subscription_id, capsys):
topic_id,
subscription_id,
["EMAIL_ADDRESS", "PHONE_NUMBER"],
timeout=420,
timeout=1
)

out, _ = capsys.readouterr()
assert "Inspection operation started" in out
# Cancel the operation
operation_id = out.split("Inspection operation started: ")[1].split("\n")[0]
operation_id = out.split(
"Inspection operation started: ")[1].split("\n")[0]
print(operation_id)
client = google.cloud.dlp_v2.DlpServiceClient()
client.cancel_dlp_job(operation_id)


def test_inspect_gcs_file_with_custom_info_types(
bucket, topic_id, subscription_id, capsys
):
bucket, topic_id, subscription_id, capsys):
dictionaries = ["gary@somedomain.com"]
regexes = ["\\(\\d{3}\\) \\d{3}-\\d{4}"]

Expand All @@ -324,41 +324,41 @@ def test_inspect_gcs_file_with_custom_info_types(
[],
custom_dictionaries=dictionaries,
custom_regexes=regexes,
timeout=420,
)
timeout=1)

out, _ = capsys.readouterr()

assert "Inspection operation started" in out
# Cancel the operation
operation_id = out.split("Inspection operation started: ")[1].split("\n")[0]
operation_id = out.split(
"Inspection operation started: ")[1].split("\n")[0]
print(operation_id)
client = google.cloud.dlp_v2.DlpServiceClient()
client.cancel_dlp_job(operation_id)


def test_inspect_gcs_file_no_results(bucket, topic_id, subscription_id, capsys):
def test_inspect_gcs_file_no_results(
bucket, topic_id, subscription_id, capsys):
inspect_content.inspect_gcs_file(
GCLOUD_PROJECT,
bucket.name,
"harmless.txt",
topic_id,
subscription_id,
["EMAIL_ADDRESS", "PHONE_NUMBER"],
timeout=420,
)
timeout=1)

out, _ = capsys.readouterr()

assert "Inspection operation started" in out
# Cancel the operation
operation_id = out.split("Inspection operation started: ")[1].split("\n")[0]
operation_id = out.split(
"Inspection operation started: ")[1].split("\n")[0]
print(operation_id)
client = google.cloud.dlp_v2.DlpServiceClient()
client.cancel_dlp_job(operation_id)


@pytest.mark.skip(reason="nondeterministically failing")
def test_inspect_gcs_image_file(bucket, topic_id, subscription_id, capsys):
inspect_content.inspect_gcs_file(
GCLOUD_PROJECT,
Expand All @@ -367,10 +367,16 @@ def test_inspect_gcs_image_file(bucket, topic_id, subscription_id, capsys):
topic_id,
subscription_id,
["EMAIL_ADDRESS", "PHONE_NUMBER"],
)
timeout=1)

out, _ = capsys.readouterr()
assert "Info type: EMAIL_ADDRESS" in out
assert "Inspection operation started" in out
# Cancel the operation
operation_id = out.split(
"Inspection operation started: ")[1].split("\n")[0]
print(operation_id)
client = google.cloud.dlp_v2.DlpServiceClient()
client.cancel_dlp_job(operation_id)


def test_inspect_gcs_multiple_files(bucket, topic_id, subscription_id, capsys):
Expand All @@ -381,55 +387,62 @@ def test_inspect_gcs_multiple_files(bucket, topic_id, subscription_id, capsys):
topic_id,
subscription_id,
["EMAIL_ADDRESS", "PHONE_NUMBER"],
)
timeout=1)

out, _ = capsys.readouterr()

assert "Inspection operation started" in out
# Cancel the operation
operation_id = out.split("Inspection operation started: ")[1].split("\n")[0]
operation_id = out.split(
"Inspection operation started: ")[1].split("\n")[0]
print(operation_id)
client = google.cloud.dlp_v2.DlpServiceClient()
client.cancel_dlp_job(operation_id)


@flaky
def test_inspect_datastore(datastore_project, topic_id, subscription_id, capsys):
@eventually_consistent.call
def _():
inspect_content.inspect_datastore(
GCLOUD_PROJECT,
datastore_project,
DATASTORE_KIND,
topic_id,
subscription_id,
["FIRST_NAME", "EMAIL_ADDRESS", "PHONE_NUMBER"],
)
def test_inspect_datastore(
datastore_project, topic_id, subscription_id, capsys):
inspect_content.inspect_datastore(
GCLOUD_PROJECT,
datastore_project,
DATASTORE_KIND,
topic_id,
subscription_id,
["FIRST_NAME", "EMAIL_ADDRESS", "PHONE_NUMBER"],
timeout=1)

out, _ = capsys.readouterr()
assert "Info type: EMAIL_ADDRESS" in out
out, _ = capsys.readouterr()
assert "Inspection operation started" in out
# Cancel the operation
operation_id = out.split(
"Inspection operation started: ")[1].split("\n")[0]
print(operation_id)
client = google.cloud.dlp_v2.DlpServiceClient()
client.cancel_dlp_job(operation_id)


@flaky
# @pytest.mark.skip(reason="too slow")
def test_inspect_datastore_no_results(
datastore_project, topic_id, subscription_id, capsys
):
@eventually_consistent.call
def _():
inspect_content.inspect_datastore(
GCLOUD_PROJECT,
datastore_project,
DATASTORE_KIND,
topic_id,
subscription_id,
["PHONE_NUMBER"],
)

out, _ = capsys.readouterr()
assert "No findings" in out


@pytest.mark.skip(reason="unknown issue")
datastore_project, topic_id, subscription_id, capsys):
inspect_content.inspect_datastore(
GCLOUD_PROJECT,
datastore_project,
DATASTORE_KIND,
topic_id,
subscription_id,
["PHONE_NUMBER"],
timeout=1)

out, _ = capsys.readouterr()
assert "Inspection operation started" in out
# Cancel the operation
operation_id = out.split(
"Inspection operation started: ")[1].split("\n")[0]
print(operation_id)
client = google.cloud.dlp_v2.DlpServiceClient()
client.cancel_dlp_job(operation_id)


def test_inspect_bigquery(bigquery_project, topic_id, subscription_id, capsys):
inspect_content.inspect_bigquery(
GCLOUD_PROJECT,
Expand All @@ -439,7 +452,13 @@ def test_inspect_bigquery(bigquery_project, topic_id, subscription_id, capsys):
topic_id,
subscription_id,
["FIRST_NAME", "EMAIL_ADDRESS", "PHONE_NUMBER"],
)
timeout=1)

out, _ = capsys.readouterr()
assert "Info type: FIRST_NAME" in out
assert "Inspection operation started" in out
# Cancel the operation
operation_id = out.split(
"Inspection operation started: ")[1].split("\n")[0]
print(operation_id)
client = google.cloud.dlp_v2.DlpServiceClient()
client.cancel_dlp_job(operation_id)
3 changes: 1 addition & 2 deletions dlp/jobs_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
# limitations under the License.

import os
from flaky import flaky

import pytest

Expand Down Expand Up @@ -66,7 +65,7 @@ def test_list_dlp_jobs(test_job_name, capsys):
assert test_job_name not in out


@flaky
@pytest.mark.flaky
def test_list_dlp_jobs_with_filter(test_job_name, capsys):
jobs.list_dlp_jobs(
GCLOUD_PROJECT,
Expand Down
1 change: 0 additions & 1 deletion dlp/requirements-test.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
pytest==5.3.2
gcp-devrel-py-tools==0.0.15
flaky==3.6.1
mock==3.0.5
24 changes: 11 additions & 13 deletions dlp/risk_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from flaky import flaky
import os
import uuid

import google.cloud.pubsub
import google.cloud.bigquery

import pytest
import os

import risk

Expand Down Expand Up @@ -160,7 +158,7 @@ def bigquery_project():
bigquery_client.delete_dataset(dataset_ref, delete_contents=True)


@flaky
@pytest.mark.flaky
def test_numerical_risk_analysis(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -178,7 +176,7 @@ def test_numerical_risk_analysis(
assert "Value Range:" in out


@flaky
@pytest.mark.flaky
def test_categorical_risk_analysis_on_string_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -197,7 +195,7 @@ def test_categorical_risk_analysis_on_string_field(
assert "Most common value occurs" in out


@flaky
@pytest.mark.flaky
def test_categorical_risk_analysis_on_number_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -215,7 +213,7 @@ def test_categorical_risk_analysis_on_number_field(
assert "Most common value occurs" in out


@flaky
@pytest.mark.flaky
def test_k_anonymity_analysis_single_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -234,7 +232,7 @@ def test_k_anonymity_analysis_single_field(
assert "Class size:" in out


@flaky
@pytest.mark.flaky
def test_k_anonymity_analysis_multiple_fields(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -253,7 +251,7 @@ def test_k_anonymity_analysis_multiple_fields(
assert "Class size:" in out


@flaky
@pytest.mark.flaky
def test_l_diversity_analysis_single_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -274,7 +272,7 @@ def test_l_diversity_analysis_single_field(
assert "Sensitive value" in out


@flaky
@pytest.mark.flaky
def test_l_diversity_analysis_multiple_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -295,7 +293,7 @@ def test_l_diversity_analysis_multiple_field(
assert "Sensitive value" in out


@flaky
@pytest.mark.flaky
def test_k_map_estimate_analysis_single_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -316,7 +314,7 @@ def test_k_map_estimate_analysis_single_field(
assert "Values" in out


@flaky
@pytest.mark.flaky
def test_k_map_estimate_analysis_multiple_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -337,7 +335,7 @@ def test_k_map_estimate_analysis_multiple_field(
assert "Values" in out


@flaky
@pytest.mark.flaky
def test_k_map_estimate_analysis_quasi_ids_info_types_equal(
topic_id, subscription_id, bigquery_project
):
Expand Down

0 comments on commit ce10217

Please sign in to comment.