Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: E2E test on endpoint versions only #10441

Merged
merged 5 commits into from
Aug 2, 2023
Merged

test: E2E test on endpoint versions only #10441

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e01ff1b
test: E2E test on endpoint versions only
kweinmeister Jul 20, 2023
e7f4f83
test on 3.8 and 3.11 based on feedback
kweinmeister Jul 24, 2023
ff520cd
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Jul 24, 2023
6db4786
Merge branch 'main' of https://github.com/GoogleCloudPlatform/python-…
kweinmeister Aug 2, 2023
2384a58
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Aug 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
🦉 Updates from OwlBot post-processor 
See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md
  • Loading branch information
@gcf-owl-bot
gcf-owl-bot[bot] committed Jul 24, 2023
commit ff520cd3df5ae0f107a3ff3f812be3500ce6c6ee
42 changes: 20 additions & 22 deletions dlp/snippets/deid_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,8 @@ def deidentify_table_with_fpe(

# Call the API.
response = dlp.deidentify_content(
request={
"parent": parent,
"deidentify_config": deidentify_config,
"item": item
})
request={"parent": parent, "deidentify_config": deidentify_config, "item": item}
)

# Print out results.
print(f"Table after de-identification: {response.item.table}")
Expand Down Expand Up @@ -194,7 +191,8 @@ def reidentify_table_with_fpe(
"parent": parent,
"reidentify_config": reidentify_config,
"item": item,
})
}
)

# Print out results.
print("Table after re-identification: {}".format(response.item.table))
Expand All @@ -212,7 +210,7 @@ def reidentify_table_with_fpe(
table_fpe_parser = subparsers.add_parser(
"deid_table_fpe",
help="Deidentify sensitive data in a string using Format Preserving "
"Encryption (FPE).",
"Encryption (FPE).",
)
table_fpe_parser.add_argument(
"project",
Expand All @@ -233,29 +231,29 @@ def reidentify_table_with_fpe(
table_fpe_parser.add_argument(
"key_name",
help="The name of the Cloud KMS key used to encrypt ('wrap') the "
"AES-256 key. Example: "
"key_name = 'projects/YOUR_GCLOUD_PROJECT/locations/YOUR_LOCATION/"
"keyRings/YOUR_KEYRING_NAME/cryptoKeys/YOUR_KEY_NAME'",
"AES-256 key. Example: "
"key_name = 'projects/YOUR_GCLOUD_PROJECT/locations/YOUR_LOCATION/"
"keyRings/YOUR_KEYRING_NAME/cryptoKeys/YOUR_KEY_NAME'",
)
table_fpe_parser.add_argument(
"wrapped_key",
help="The encrypted ('wrapped') AES-256 key to use. This key should "
"be encrypted using the Cloud KMS key specified by key_name.",
"be encrypted using the Cloud KMS key specified by key_name.",
)
table_fpe_parser.add_argument(
"-a",
"--alphabet",
default="ALPHA_NUMERIC",
help="The set of characters to replace sensitive ones with. Commonly "
'used subsets of the alphabet include "NUMERIC", "HEXADECIMAL", '
'"UPPER_CASE_ALPHA_NUMERIC", "ALPHA_NUMERIC", '
'"FFX_COMMON_NATIVE_ALPHABET_UNSPECIFIED"',
'used subsets of the alphabet include "NUMERIC", "HEXADECIMAL", '
'"UPPER_CASE_ALPHA_NUMERIC", "ALPHA_NUMERIC", '
'"FFX_COMMON_NATIVE_ALPHABET_UNSPECIFIED"',
)

reid_table_fpe_parser = subparsers.add_parser(
"reid_table_fpe",
help="Re-identify sensitive data in a table using Format Preserving "
"Encryption (FPE).",
"Encryption (FPE).",
)
reid_table_fpe_parser.add_argument(
"project",
Expand All @@ -276,23 +274,23 @@ def reidentify_table_with_fpe(
reid_table_fpe_parser.add_argument(
"key_name",
help="The name of the Cloud KMS key used to encrypt ('wrap') the "
"AES-256 key. Example: "
"key_name = 'projects/YOUR_GCLOUD_PROJECT/locations/YOUR_LOCATION/"
"keyRings/YOUR_KEYRING_NAME/cryptoKeys/YOUR_KEY_NAME'",
"AES-256 key. Example: "
"key_name = 'projects/YOUR_GCLOUD_PROJECT/locations/YOUR_LOCATION/"
"keyRings/YOUR_KEYRING_NAME/cryptoKeys/YOUR_KEY_NAME'",
)
reid_table_fpe_parser.add_argument(
"wrapped_key",
help="The encrypted ('wrapped') AES-256 key to use. This key should "
"be encrypted using the Cloud KMS key specified by key_name.",
"be encrypted using the Cloud KMS key specified by key_name.",
)
reid_table_fpe_parser.add_argument(
"-a",
"--alphabet",
default="ALPHA_NUMERIC",
help="The set of characters to replace sensitive ones with. Commonly "
'used subsets of the alphabet include "NUMERIC", "HEXADECIMAL", '
'"UPPER_CASE_ALPHA_NUMERIC", "ALPHA_NUMERIC", '
'"FFX_COMMON_NATIVE_ALPHABET_UNSPECIFIED"',
'used subsets of the alphabet include "NUMERIC", "HEXADECIMAL", '
'"UPPER_CASE_ALPHA_NUMERIC", "ALPHA_NUMERIC", '
'"FFX_COMMON_NATIVE_ALPHABET_UNSPECIFIED"',
)

args = parser.parse_args()
Expand Down
22 changes: 13 additions & 9 deletions dlp/snippets/deid_table_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,22 +31,24 @@
)


def test_deidentify_and_reidentify_table_with_fpe(capsys: pytest.CaptureFixture) -> None:
def test_deidentify_and_reidentify_table_with_fpe(
capsys: pytest.CaptureFixture,
) -> None:
table_data = {
"header": ["employee_id", "date", "compensation"],
"rows": [
["11111", "2015", "$10"],
["22222", "2016", "$20"],
["33333", "2016", "$15"],
]
],
}

deid_table.deidentify_table_with_fpe(
GCLOUD_PROJECT,
table_data["header"],
table_data["rows"],
["employee_id"],
alphabet='NUMERIC',
alphabet="NUMERIC",
wrapped_key=base64.b64decode(WRAPPED_KEY),
key_name=KEY_NAME,
)
Expand All @@ -58,21 +60,23 @@ def test_deidentify_and_reidentify_table_with_fpe(capsys: pytest.CaptureFixture)
response = out.split(":")[1:]

deid_col_id = response.index(' "employee_id"\n}\nheaders {\n name')
total_columns = len(table_data['header'])
total_rows = len(table_data['rows'][0])
total_columns = len(table_data["header"])
total_rows = len(table_data["rows"][0])

deid_emp_ids = [response[i].split("\n")[0][2:-1] for i in
range(deid_col_id + total_columns, len(response), total_columns)]
deid_emp_ids = [
response[i].split("\n")[0][2:-1]
for i in range(deid_col_id + total_columns, len(response), total_columns)
]

for i in range(total_rows):
table_data['rows'][i][deid_col_id - 1] = deid_emp_ids[i]
table_data["rows"][i][deid_col_id - 1] = deid_emp_ids[i]

deid_table.reidentify_table_with_fpe(
GCLOUD_PROJECT,
table_data["header"],
table_data["rows"],
["employee_id"],
alphabet='NUMERIC',
alphabet="NUMERIC",
wrapped_key=base64.b64decode(WRAPPED_KEY),
key_name=KEY_NAME,
)
Expand Down
23 changes: 7 additions & 16 deletions dlp/snippets/inspect_content.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1590,21 +1590,13 @@ def inspect_gcs_send_to_scc(
inspect_config = {
"info_types": info_types,
"min_likelihood": google.cloud.dlp_v2.Likelihood.UNLIKELY,
"limits": {
"max_findings_per_request": max_findings
},
"limits": {"max_findings_per_request": max_findings},
"include_quote": True,
}

# Construct a cloud_storage_options dictionary with the bucket's URL.
url = f"gs://{bucket}"
storage_config = {
"cloud_storage_options": {
"file_set": {
"url": url
}
}
}
storage_config = {"cloud_storage_options": {"file_set": {"url": url}}}

# Tell the API where to send a notification when the job is complete.
actions = [{"publish_summary_to_cscc": {}}]
Expand Down Expand Up @@ -1639,7 +1631,7 @@ def inspect_gcs_send_to_scc(
if job.state == google.cloud.dlp_v2.DlpJob.JobState.DONE:
break
elif job.state == google.cloud.dlp_v2.DlpJob.JobState.FAILED:
print('Job Failed, Please check the configuration.')
print("Job Failed, Please check the configuration.")
return

# Sleep for a short duration before checking the job status again.
Expand All @@ -1657,6 +1649,7 @@ def inspect_gcs_send_to_scc(
else:
print("No findings.")


# [END dlp_inspect_gcs_send_to_scc]


Expand All @@ -1675,7 +1668,6 @@ def inspect_datastore_send_to_scc(
namespace_id: str = None,
max_findings: int = 100,
) -> None:

"""
Uses the Data Loss Prevention API to inspect Datastore data and
send the results to Google Security Command Center.
Expand Down Expand Up @@ -1711,7 +1703,7 @@ def inspect_datastore_send_to_scc(
"project_id": datastore_project,
"namespace_id": namespace_id,
},
"kind": {"name": kind}
"kind": {"name": kind},
}
}

Expand Down Expand Up @@ -1748,7 +1740,7 @@ def inspect_datastore_send_to_scc(
if job.state == google.cloud.dlp_v2.DlpJob.JobState.DONE:
break
elif job.state == google.cloud.dlp_v2.DlpJob.JobState.FAILED:
print('Job Failed, Please check the configuration.')
print("Job Failed, Please check the configuration.")
return

# Sleep for a short duration before checking the job status again.
Expand Down Expand Up @@ -1780,7 +1772,6 @@ def inspect_bigquery_send_to_scc(
info_types: List[str],
max_findings: int = 100,
) -> None:

"""
Uses the Data Loss Prevention API to inspect public bigquery dataset
and send the results to Google Security Command Center.
Expand Down Expand Up @@ -1849,7 +1840,7 @@ def inspect_bigquery_send_to_scc(
if job.state == google.cloud.dlp_v2.DlpJob.JobState.DONE:
break
elif job.state == google.cloud.dlp_v2.DlpJob.JobState.FAILED:
print('Job Failed, Please check the configuration.')
print("Job Failed, Please check the configuration.")
return

# Sleep for a short duration before checking the job status again.
Expand Down
29 changes: 15 additions & 14 deletions dlp/snippets/inspect_content_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -697,18 +697,18 @@ def test_inspect_data_to_hybrid_job_trigger(

@mock.patch("google.cloud.dlp_v2.DlpServiceClient")
def test_inspect_gcs_send_to_scc(
dlp_client: MagicMock,
capsys: pytest.CaptureFixture
dlp_client: MagicMock, capsys: pytest.CaptureFixture
) -> None:

# Configure the mock DLP client and its behavior.
mock_dlp_instance = dlp_client.return_value
# Configure the mock CreateDlpJob DLP method and its behavior.
mock_dlp_instance.create_dlp_job.return_value.name = f'projects/{GCLOUD_PROJECT}/dlpJobs/test_job'
mock_dlp_instance.create_dlp_job.return_value.name = (
f"projects/{GCLOUD_PROJECT}/dlpJobs/test_job"
)

# Configure the mock GetDlpJob DLP method and its behavior.
mock_job = mock_dlp_instance.get_dlp_job.return_value
mock_job.name = f'projects/{GCLOUD_PROJECT}/dlpJobs/test_job'
mock_job.name = f"projects/{GCLOUD_PROJECT}/dlpJobs/test_job"
mock_job.state = google.cloud.dlp_v2.DlpJob.JobState.DONE

file = open(os.path.join(RESOURCE_DIRECTORY, "test.txt"), "r")
Expand Down Expand Up @@ -742,18 +742,18 @@ def test_inspect_gcs_send_to_scc(

@mock.patch("google.cloud.dlp_v2.DlpServiceClient")
def test_inspect_bigquery_send_to_scc(
dlp_client: MagicMock,
capsys: pytest.CaptureFixture
dlp_client: MagicMock, capsys: pytest.CaptureFixture
) -> None:

# Configure the mock DLP client and its behavior.
mock_dlp_instance = dlp_client.return_value
# Configure the mock CreateDlpJob DLP method and its behavior.
mock_dlp_instance.create_dlp_job.return_value.name = f'projects/{GCLOUD_PROJECT}/dlpJobs/test_job'
mock_dlp_instance.create_dlp_job.return_value.name = (
f"projects/{GCLOUD_PROJECT}/dlpJobs/test_job"
)

# Configure the mock GetDlpJob DLP method and its behavior.
mock_job = mock_dlp_instance.get_dlp_job.return_value
mock_job.name = f'projects/{GCLOUD_PROJECT}/dlpJobs/test_job'
mock_job.name = f"projects/{GCLOUD_PROJECT}/dlpJobs/test_job"

mock_job.state = google.cloud.dlp_v2.DlpJob.JobState.DONE

Expand All @@ -779,17 +779,18 @@ def test_inspect_bigquery_send_to_scc(

@mock.patch("google.cloud.dlp_v2.DlpServiceClient")
def test_inspect_datastore_send_to_scc(
dlp_client: MagicMock,
capsys: pytest.CaptureFixture
dlp_client: MagicMock, capsys: pytest.CaptureFixture
) -> None:
# Configure the mock DLP client and its behavior.
mock_dlp_instance = dlp_client.return_value
# Configure the mock CreateDlpJob DLP method and its behavior.
mock_dlp_instance.create_dlp_job.return_value.name = f'projects/{GCLOUD_PROJECT}/dlpJobs/test_job'
mock_dlp_instance.create_dlp_job.return_value.name = (
f"projects/{GCLOUD_PROJECT}/dlpJobs/test_job"
)

# Configure the mock GetDlpJob DLP method and its behavior.
mock_job = mock_dlp_instance.get_dlp_job.return_value
mock_job.name = f'projects/{GCLOUD_PROJECT}/dlpJobs/test_job'
mock_job.name = f"projects/{GCLOUD_PROJECT}/dlpJobs/test_job"
mock_job.state = google.cloud.dlp_v2.DlpJob.JobState.DONE

mock_job.inspect_details.result.info_type_stats.info_type.name = "EMAIL_ADDRESS"
Expand Down
4 changes: 3 additions & 1 deletion secretmanager/snippets/consume_event_notification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ def consume_event_notification(event: dict, unused_context: None) -> str:
event_type = event["attributes"]["eventType"]
secret_id = event["attributes"]["secretId"]
secret_metadata = base64.b64decode(event["data"]).decode("utf-8")
event_notification = f"Received {event_type} for {secret_id}. New metadata: {secret_metadata}"
event_notification = (
f"Received {event_type} for {secret_id}. New metadata: {secret_metadata}"
)
print(event_notification)
return event_notification

Expand Down