Add connection key to execution log (#2100)

Co-authored-by: Paul Sanders <pau@ethyca.com>

.fides/db_dataset.yml

@@ -1053,6 +1053,8 @@ dataset:
     - name: created_at
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: connection_key
+      data_categories: [system.operations]
     - name: dataset_name
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified

CHANGELOG.md

@@ -19,6 +19,7 @@ The types of changes are:
 
 ### Added
 
+* Added the connection key to the execution log [#2100](https://github.com/ethyca/fides/pull/2100)
 * Added endpoints to retrieve DSR `Rule`s and `Rule Target`s [#2116](https://github.com/ethyca/fides/pull/2116)
 
 ### Fixed

src/fides/api/ctl/migrations/versions/3caf11127442_add_connection_key_to_execution_log.py

@@ -0,0 +1,36 @@
+"""Add connection key to execution log
+
+Revision ID: 3caf11127442
+Revises: 1f61c765cd1c
+Create Date: 2022-12-21 13:36:17.859959
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "3caf11127442"
+down_revision = "1f61c765cd1c"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "executionlog", sa.Column("connection_key", sa.String(), nullable=True)
+    )
+    op.create_index(
+        op.f("ix_executionlog_connection_key"),
+        "executionlog",
+        ["connection_key"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_executionlog_connection_key"), table_name="executionlog")
+    op.drop_column("executionlog", "connection_key")
+    # ### end Alembic commands ###

src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py

@@ -327,6 +327,7 @@ def execution_and_audit_logs_by_dataset_name(
         ExecutionLog.privacy_request_id,
         ExecutionLog.dataset_name,
         ExecutionLog.collection_name,
+        ExecutionLog.connection_key,
         ExecutionLog.fields_affected,
         ExecutionLog.action_type,
         null().label("user_id"),
@@ -341,6 +342,7 @@ def execution_and_audit_logs_by_dataset_name(
         AuditLog.privacy_request_id,
         null().label("dataset_name"),
         null().label("collection_name"),
+        null().label("connection_key"),
         null().label("fields_affected"),
         null().label("action_type"),
         AuditLog.user_id,

src/fides/api/ops/models/privacy_request.py

@@ -938,6 +938,7 @@ class ExecutionLog(Base):
     generated by the query builder.
     """
 
+    connection_key = Column(String, index=True)
     # Name of the fides-annotated dataset, for example: my-mongo-db
     dataset_name = Column(String, index=True)
     # Name of the particular collection or table affected

src/fides/api/ops/schemas/privacy_request.py

@@ -109,13 +109,15 @@ class Config:
 class ExecutionLogDetailResponse(ExecutionLogResponse):
     """Schema for the detailed ExecutionLogs when accessed directly"""
 
+    connection_key: Optional[str]
     dataset_name: Optional[str]
 
 
 class ExecutionAndAuditLogResponse(BaseSchema):
     """Schema for the combined ExecutionLogs and Audit Logs
     associated with a PrivacyRequest"""
 
+    connection_key: Optional[str]
     collection_name: Optional[str]
     fields_affected: Optional[List[FieldsAffectedResponse]]
     message: Optional[str]

src/fides/api/ops/task/graph_task.py

@@ -337,6 +337,7 @@ def update_status(
     ) -> None:
         """Update status activities"""
         self.resources.write_execution_log(
+            self.traversal_node.node.dataset.connection_key,
             self.traversal_node.address,
             fields_affected,
             action_type,

src/fides/api/ops/task/task_resources.py

@@ -160,6 +160,7 @@ def get_all_cached_erasures(self) -> Dict[str, int]:
 
     def write_execution_log(  # pylint: disable=too-many-arguments
         self,
+        connection_key: str,
         collection_address: CollectionAddress,
         fields_affected: Any,
         action_type: ActionType,
@@ -172,6 +173,7 @@ def write_execution_log(  # pylint: disable=too-many-arguments
         ExecutionLog.create(
             db=db,
             data={
+                "connection_key": connection_key,
                 "dataset_name": collection_address.dataset,
                 "collection_name": collection_address.collection,
                 "fields_affected": fields_affected,

tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py

@@ -1131,6 +1131,7 @@ def test_verbose_privacy_requests(
                     "results": {
                         "Request approved": [
                             {
+                                "connection_key": None,
                                 "collection_name": None,
                                 "fields_affected": None,
                                 "message": "",
@@ -1142,6 +1143,7 @@ def test_verbose_privacy_requests(
                         ],
                         "my-mongo-db": [
                             {
+                                "connection_key": None,
                                 "collection_name": "orders",
                                 "fields_affected": [
                                     {
@@ -1161,6 +1163,7 @@ def test_verbose_privacy_requests(
                         ],
                         "my-postgres-db": [
                             {
+                                "connection_key": None,
                                 "collection_name": "user",
                                 "fields_affected": [
                                     {
@@ -1178,6 +1181,7 @@ def test_verbose_privacy_requests(
                                 "user_id": None,
                             },
                             {
+                                "connection_key": None,
                                 "collection_name": "address",
                                 "fields_affected": [
                                     {
@@ -1225,6 +1229,7 @@ def test_verbose_privacy_request_embed_limit(
             ExecutionLog.create(
                 db=db,
                 data={
+                    "connection_key": "my-postgres-db-key",
                     "dataset_name": "my-postgres-db",
                     "collection_name": f"test_collection_{i}",
                     "fields_affected": [],
@@ -1609,6 +1614,7 @@ def test_get_execution_logs(
                     "action_type": "access",
                     "status": "pending",
                     "updated_at": stringify_date(postgres_execution_log.updated_at),
+                    "connection_key": None,
                     "dataset_name": "my-postgres-db",
                 },
                 {
@@ -1624,6 +1630,7 @@ def test_get_execution_logs(
                     "action_type": "access",
                     "status": "in_processing",
                     "updated_at": stringify_date(mongo_execution_log.updated_at),
+                    "connection_key": None,
                     "dataset_name": "my-mongo-db",
                 },
                 {
@@ -1646,6 +1653,7 @@ def test_get_execution_logs(
                     "updated_at": stringify_date(
                         second_postgres_execution_log.updated_at
                     ),
+                    "connection_key": None,
                     "dataset_name": "my-postgres-db",
                 },
             ],