From 03a3aab4d08d516f003587d31a2c4043e9cda09f Mon Sep 17 00:00:00 2001 From: Yuval Yaron <43217306+yuvalyaron@users.noreply.github.com> Date: Wed, 14 Sep 2022 14:04:32 +0300 Subject: [PATCH] Rename Azure Function ToDeleteTrigger to DataDeletionTrigger (#2592) * add 'previous status' field to 'status changed' message * add support for container deletion in azure function 'toDeleteTrigger' * handle request cancellation in StatusChangedQueueTrigger * fix output event names * update versions * clean code by extracting to methods * update changelog * fix unit tests * add unit tests * use already declared variables instead of request_properties * update api version * update changelog Co-authored-by: Elad Iwanir <13205761+eladiw@users.noreply.github.com> * update log message Co-authored-by: Elad Iwanir <13205761+eladiw@users.noreply.github.com> * rename references of toDelete event to dataDeletion event in statusChanged function * change toDelete to DataDeletion * update version and changelog * rename ToDelete to DataDeletion * update version * fix terraform file format * fix terraform variable values * update core version Co-authored-by: Elad Iwanir <13205761+eladiw@users.noreply.github.com> --- .../BlobCreatedTrigger/__init__.py | 4 +-- .../BlobCreatedTrigger/function.json | 6 ++-- .../__init__.py | 2 +- .../function.json | 2 +- .../StatusChangedQueueTrigger/function.json | 4 +-- airlock_processor/_version.py | 2 +- ...igger.py => test_data_deletion_trigger.py} | 10 +++---- .../terraform/airlock/airlock_processor.tf | 30 +++++++++---------- .../terraform/airlock/eventgrid_topics.tf | 28 ++++++++--------- templates/core/terraform/airlock/locals.tf | 6 ++-- .../core/terraform/airlock/service_bus.tf | 4 +-- templates/core/version.txt | 2 +- 12 files changed, 50 insertions(+), 50 deletions(-) rename airlock_processor/{ToDeleteTrigger => DataDeletionTrigger}/__init__.py (95%) rename airlock_processor/{ToDeleteTrigger => DataDeletionTrigger}/function.json (79%) rename airlock_processor/tests/{test_to_delete_trigger.py => test_data_deletion_trigger.py} (84%) diff --git a/airlock_processor/BlobCreatedTrigger/__init__.py b/airlock_processor/BlobCreatedTrigger/__init__.py index 680488aaa0..b55d1ac583 100644 --- a/airlock_processor/BlobCreatedTrigger/__init__.py +++ b/airlock_processor/BlobCreatedTrigger/__init__.py @@ -14,7 +14,7 @@ def main(msg: func.ServiceBusMessage, stepResultEvent: func.Out[func.EventGridOutputEvent], - toDeleteEvent: func.Out[func.EventGridOutputEvent]): + dataDeletionEvent: func.Out[func.EventGridOutputEvent]): logging.info("Python ServiceBus topic trigger processed message - A new blob was created!.") body = msg.get_body().decode('utf-8') @@ -74,7 +74,7 @@ def main(msg: func.ServiceBusMessage, logging.info(f"copied from history: {copied_from}") # signal that the container where we copied from can now be deleted - toDeleteEvent.set( + dataDeletionEvent.set( func.EventGridOutputEvent( id=str(uuid.uuid4()), data={"blob_to_delete": copied_from[-1]}, # last container in copied_from is the one we just copied from diff --git a/airlock_processor/BlobCreatedTrigger/function.json b/airlock_processor/BlobCreatedTrigger/function.json index 1dcedd7925..5bde252c39 100644 --- a/airlock_processor/BlobCreatedTrigger/function.json +++ b/airlock_processor/BlobCreatedTrigger/function.json @@ -19,9 +19,9 @@ }, { "type": "eventGrid", - "name": "toDeleteEvent", - "topicEndpointUri": "EVENT_GRID_TO_DELETE_TOPIC_URI_SETTING", - "topicKeySetting": "EVENT_GRID_TO_DELETE_TOPIC_KEY_SETTING", + "name": "dataDeletionEvent", + "topicEndpointUri": "EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING", + "topicKeySetting": "EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING", "direction": "out" } ] diff --git a/airlock_processor/ToDeleteTrigger/__init__.py b/airlock_processor/DataDeletionTrigger/__init__.py similarity index 95% rename from airlock_processor/ToDeleteTrigger/__init__.py rename to airlock_processor/DataDeletionTrigger/__init__.py index a82a83973c..581094981d 100644 --- a/airlock_processor/ToDeleteTrigger/__init__.py +++ b/airlock_processor/DataDeletionTrigger/__init__.py @@ -21,7 +21,7 @@ def delete_blob_and_container_if_last_blob(blob_url: str): return # If it's the only blob in the container, we need to delete the container too - # Check how many blobs are in the container (note: this exausts the generator) + # Check how many blobs are in the container (note: this exhausts the generator) blobs_num = sum(1 for _ in container_client.list_blobs()) logging.info(f'Found {blobs_num} blobs in the container') diff --git a/airlock_processor/ToDeleteTrigger/function.json b/airlock_processor/DataDeletionTrigger/function.json similarity index 79% rename from airlock_processor/ToDeleteTrigger/function.json rename to airlock_processor/DataDeletionTrigger/function.json index 951d048c39..2b2bb580da 100644 --- a/airlock_processor/ToDeleteTrigger/function.json +++ b/airlock_processor/DataDeletionTrigger/function.json @@ -6,7 +6,7 @@ "name": "msg", "type": "serviceBusTrigger", "direction": "in", - "queueName": "%AIRLOCK_TO_DELETE_QUEUE_NAME%", + "queueName": "%AIRLOCK_DATA_DELETION_QUEUE_NAME%", "connection": "SB_CONNECTION_STRING" } ] diff --git a/airlock_processor/StatusChangedQueueTrigger/function.json b/airlock_processor/StatusChangedQueueTrigger/function.json index 977644ef6f..c5e7be3356 100644 --- a/airlock_processor/StatusChangedQueueTrigger/function.json +++ b/airlock_processor/StatusChangedQueueTrigger/function.json @@ -18,8 +18,8 @@ { "type": "eventGrid", "name": "dataDeletionEvent", - "topicEndpointUri": "EVENT_GRID_TO_DELETE_TOPIC_URI_SETTING", - "topicKeySetting": "EVENT_GRID_TO_DELETE_TOPIC_KEY_SETTING", + "topicEndpointUri": "EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING", + "topicKeySetting": "EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING", "direction": "out" } ] diff --git a/airlock_processor/_version.py b/airlock_processor/_version.py index a34b2f6b04..a3a9bd5443 100644 --- a/airlock_processor/_version.py +++ b/airlock_processor/_version.py @@ -1 +1 @@ -__version__ = "0.4.7" +__version__ = "0.4.8" diff --git a/airlock_processor/tests/test_to_delete_trigger.py b/airlock_processor/tests/test_data_deletion_trigger.py similarity index 84% rename from airlock_processor/tests/test_to_delete_trigger.py rename to airlock_processor/tests/test_data_deletion_trigger.py index 4a812b952f..a34bc9a930 100644 --- a/airlock_processor/tests/test_to_delete_trigger.py +++ b/airlock_processor/tests/test_data_deletion_trigger.py @@ -1,11 +1,11 @@ from unittest import TestCase from unittest.mock import MagicMock, patch -from ToDeleteTrigger import delete_blob_and_container_if_last_blob +from DataDeletionTrigger import delete_blob_and_container_if_last_blob -class TestToDeleteTrigger(TestCase): - @patch("ToDeleteTrigger.BlobServiceClient") +class TestDataDeletionTrigger(TestCase): + @patch("DataDeletionTrigger.BlobServiceClient") def test_delete_blob_and_container_if_last_blob_deletes_container(self, mock_blob_service_client): blob_url = "https://stalimextest.blob.core.windows.net/c144728c-3c69-4a58-afec-48c2ec8bfd45/test_dataset.txt" @@ -15,7 +15,7 @@ def test_delete_blob_and_container_if_last_blob_deletes_container(self, mock_blo mock_blob_service_client().get_container_client().delete_container.assert_called_once() - @patch("ToDeleteTrigger.BlobServiceClient") + @patch("DataDeletionTrigger.BlobServiceClient") def test_delete_blob_and_container_if_last_blob_doesnt_delete_container(self, mock_blob_service_client): blob_url = "https://stalimextest.blob.core.windows.net/c144728c-3c69-4a58-afec-48c2ec8bfd45/test_dataset.txt" @@ -25,7 +25,7 @@ def test_delete_blob_and_container_if_last_blob_doesnt_delete_container(self, mo mock_blob_service_client().get_container_client().delete_container.assert_not_called() - @patch("ToDeleteTrigger.BlobServiceClient") + @patch("DataDeletionTrigger.BlobServiceClient") def test_delete_blob_and_container_if_last_blob_deletes_container_if_no_blob_specified(self, mock_blob_service_client): blob_url = "https://stalimextest.blob.core.windows.net/c144728c-3c69-4a58-afec-48c2ec8bfd45/" delete_blob_and_container_if_last_blob(blob_url) diff --git a/templates/core/terraform/airlock/airlock_processor.tf b/templates/core/terraform/airlock/airlock_processor.tf index 348486f519..b7ec9bb62a 100644 --- a/templates/core/terraform/airlock/airlock_processor.tf +++ b/templates/core/terraform/airlock/airlock_processor.tf @@ -47,21 +47,21 @@ resource "azurerm_linux_function_app" "airlock_function_app" { } app_settings = { - "SB_CONNECTION_STRING" = var.airlock_servicebus.default_primary_connection_string - "BLOB_CREATED_TOPIC_NAME" = azurerm_servicebus_topic.blob_created.name - "TOPIC_SUBSCRIPTION_NAME" = azurerm_servicebus_subscription.airlock_processor.name - "EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.step_result.endpoint - "EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.step_result.primary_access_key - "EVENT_GRID_TO_DELETE_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.to_delete.endpoint - "EVENT_GRID_TO_DELETE_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.to_delete.primary_access_key - "WEBSITES_ENABLE_APP_SERVICE_STORAGE" = false - "AIRLOCK_STATUS_CHANGED_QUEUE_NAME" = local.status_changed_queue_name - "AIRLOCK_SCAN_RESULT_QUEUE_NAME" = local.scan_result_queue_name - "AIRLOCK_TO_DELETE_QUEUE_NAME" = local.to_delete_queue_name - "ENABLE_MALWARE_SCANNING" = var.enable_malware_scanning - "MANAGED_IDENTITY_CLIENT_ID" = azurerm_user_assigned_identity.airlock_id.client_id - "TRE_ID" = var.tre_id - "WEBSITE_CONTENTOVERVNET" = 1 + "SB_CONNECTION_STRING" = var.airlock_servicebus.default_primary_connection_string + "BLOB_CREATED_TOPIC_NAME" = azurerm_servicebus_topic.blob_created.name + "TOPIC_SUBSCRIPTION_NAME" = azurerm_servicebus_subscription.airlock_processor.name + "EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.step_result.endpoint + "EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.step_result.primary_access_key + "EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.data_deletion.endpoint + "EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.data_deletion.primary_access_key + "WEBSITES_ENABLE_APP_SERVICE_STORAGE" = false + "AIRLOCK_STATUS_CHANGED_QUEUE_NAME" = local.status_changed_queue_name + "AIRLOCK_SCAN_RESULT_QUEUE_NAME" = local.scan_result_queue_name + "AIRLOCK_DATA_DELETION_QUEUE_NAME" = local.data_deletion_queue_name + "ENABLE_MALWARE_SCANNING" = var.enable_malware_scanning + "MANAGED_IDENTITY_CLIENT_ID" = azurerm_user_assigned_identity.airlock_id.client_id + "TRE_ID" = var.tre_id + "WEBSITE_CONTENTOVERVNET" = 1 } site_config { diff --git a/templates/core/terraform/airlock/eventgrid_topics.tf b/templates/core/terraform/airlock/eventgrid_topics.tf index e29919331c..f6287f2746 100644 --- a/templates/core/terraform/airlock/eventgrid_topics.tf +++ b/templates/core/terraform/airlock/eventgrid_topics.tf @@ -113,8 +113,8 @@ resource "azurerm_private_endpoint" "eg_status_changed" { } } -resource "azurerm_eventgrid_topic" "to_delete" { - name = local.to_delete_topic_name +resource "azurerm_eventgrid_topic" "data_deletion" { + name = local.data_deletion_topic_name location = var.location resource_group_name = var.resource_group_name public_network_access_enabled = var.enable_local_debugging @@ -130,18 +130,18 @@ resource "azurerm_eventgrid_topic" "to_delete" { lifecycle { ignore_changes = [tags] } } -resource "azurerm_role_assignment" "servicebus_sender_to_delete" { +resource "azurerm_role_assignment" "servicebus_sender_data_deletion" { scope = var.airlock_servicebus.id role_definition_name = "Azure Service Bus Data Sender" - principal_id = azurerm_eventgrid_topic.to_delete.identity.0.principal_id + principal_id = azurerm_eventgrid_topic.data_deletion.identity.0.principal_id depends_on = [ - azurerm_eventgrid_topic.to_delete + azurerm_eventgrid_topic.data_deletion ] } -resource "azurerm_private_endpoint" "eg_to_delete" { - name = "pe-eg-to-delete-${var.tre_id}" +resource "azurerm_private_endpoint" "eg_data_deletion" { + name = "pe-eg-data-deletion-${var.tre_id}" location = var.location resource_group_name = var.resource_group_name subnet_id = var.airlock_events_subnet_id @@ -155,7 +155,7 @@ resource "azurerm_private_endpoint" "eg_to_delete" { private_service_connection { name = "psc-eg-${var.tre_id}" - private_connection_resource_id = azurerm_eventgrid_topic.to_delete.id + private_connection_resource_id = azurerm_eventgrid_topic.data_deletion.id is_manual_connection = false subresource_names = ["topic"] } @@ -379,19 +379,19 @@ resource "azurerm_eventgrid_event_subscription" "status_changed" { ] } -resource "azurerm_eventgrid_event_subscription" "to_delete" { - name = local.to_delete_eventgrid_subscription_name - scope = azurerm_eventgrid_topic.to_delete.id +resource "azurerm_eventgrid_event_subscription" "data_deletion" { + name = local.data_deletion_eventgrid_subscription_name + scope = azurerm_eventgrid_topic.data_deletion.id - service_bus_queue_endpoint_id = azurerm_servicebus_queue.to_delete.id + service_bus_queue_endpoint_id = azurerm_servicebus_queue.data_deletion.id delivery_identity { type = "SystemAssigned" } depends_on = [ - azurerm_eventgrid_topic.to_delete, - azurerm_role_assignment.servicebus_sender_to_delete + azurerm_eventgrid_topic.data_deletion, + azurerm_role_assignment.servicebus_sender_data_deletion ] } diff --git a/templates/core/terraform/airlock/locals.tf b/templates/core/terraform/airlock/locals.tf index 28cecf6cbe..95d443d054 100644 --- a/templates/core/terraform/airlock/locals.tf +++ b/templates/core/terraform/airlock/locals.tf @@ -22,19 +22,19 @@ locals { step_result_topic_name = "evgt-airlock-step-result-${local.topic_name_suffix}" status_changed_topic_name = "evgt-airlock-status-changed-${local.topic_name_suffix}" notification_topic_name = "evgt-airlock-notification-${local.topic_name_suffix}" - to_delete_topic_name = "evgt-airlock-to-delete-${local.topic_name_suffix}" + data_deletion_topic_name = "evgt-airlock-data-deletion-${local.topic_name_suffix}" step_result_queue_name = "airlock-step-result" status_changed_queue_name = "airlock-status-changed" scan_result_queue_name = "airlock-scan-result" - to_delete_queue_name = "airlock-to-delete" + data_deletion_queue_name = "airlock-data-deletion" blob_created_topic_name = "airlock-blob-created" blob_created_al_processor_subscription_name = "airlock-blob-created-airlock-processor" step_result_eventgrid_subscription_name = "evgs-airlock-update-status" status_changed_eventgrid_subscription_name = "evgs-airlock-status-changed" - to_delete_eventgrid_subscription_name = "evgs-airlock-to-delete" + data_deletion_eventgrid_subscription_name = "evgs-airlock-data-deletion" import_inprogress_eventgrid_subscription_name = "evgs-airlock-import-in-progress-blob-created" import_rejected_eventgrid_subscription_name = "evgs-airlock-import-rejected-blob-created" import_blocked_eventgrid_subscription_name = "evgs-airlock-import-blocked-blob-created" diff --git a/templates/core/terraform/airlock/service_bus.tf b/templates/core/terraform/airlock/service_bus.tf index 4855451ea2..82e34e86a1 100644 --- a/templates/core/terraform/airlock/service_bus.tf +++ b/templates/core/terraform/airlock/service_bus.tf @@ -20,8 +20,8 @@ resource "azurerm_servicebus_queue" "scan_result" { enable_partitioning = false } -resource "azurerm_servicebus_queue" "to_delete" { - name = local.to_delete_queue_name +resource "azurerm_servicebus_queue" "data_deletion" { + name = local.data_deletion_queue_name namespace_id = var.airlock_servicebus.id enable_partitioning = false diff --git a/templates/core/version.txt b/templates/core/version.txt index c6dd1e2c0a..9e6207df0a 100644 --- a/templates/core/version.txt +++ b/templates/core/version.txt @@ -1 +1 @@ -__version__ = "0.4.23" +__version__ = "0.4.24"