Bug 1697602 Remove derived tables for AET (#1894)

* Bug 1697602 Remove derived tables for AET

* Update GRAVEYARD doc

Author: jklukas

GRAVEYARD.md

@@ -2,6 +2,14 @@
 
 This document records interesting code that we've deleted for the sake of discoverability for the future.
 
+## 2021-03 Account Ecosystem Telemetry (AET) derived tables
+
+- [Removal PR](https://github.com/mozilla/bigquery-etl/pull/1894)
+
+AET was never released except for a short test in the beta population,
+and now the project has been decommissioned, so there is no longer
+any need for these derived tables.
+
 ## 2020-04 Fenix baseline_daily and clients_last_seen
 
 - [Removal PR](https://github.com/mozilla/bigquery-etl/pull/925)

bigquery_etl/shredder/config.py

@@ -72,9 +72,6 @@ def fields(self) -> Tuple[str, ...]:
 USER_ID = "user_id"
 POCKET_ID = "pocket_id"
 SHIELD_ID = "shield_id"
-ECOSYSTEM_CLIENT_ID = "ecosystem_client_id"
-ECOSYSTEM_CLIENT_ID_HASH = f"{ECOSYSTEM_CLIENT_ID}_hash"
-DESKTOP_ECOSYSTEM_CLIENT_ID = f"payload.{ECOSYSTEM_CLIENT_ID}"
 PIONEER_ID = "pioneer_id"
 ID = "id"
 CFR_ID = f"COALESCE({CLIENT_ID}, {IMPRESSION_ID})"
@@ -95,14 +92,6 @@ def fields(self) -> Tuple[str, ...]:
     f" UNNEST([{CLIENT_ID}, {IMPRESSION_SRC.field}]) AS `_",
     field="_",
 )
-ECOSYSTEM_CLIENT_ID_HMAC_SRC = DeleteSource(
-    table="account_ecosystem_restricted.ecosystem_client_id_deletion_v1",
-    field=ECOSYSTEM_CLIENT_ID_HASH,
-)
-ECOSYSTEM_CLIENT_ID_SRC = DeleteSource(
-    table="account_ecosystem_restricted.ecosystem_client_id_deletion_v1",
-    field=ECOSYSTEM_CLIENT_ID,
-)
 FXA_HMAC_SRC = DeleteSource(
     table="firefox_accounts_derived.fxa_delete_events_v1", field="hmac_user_id"
 )
@@ -138,8 +127,6 @@ def fields(self) -> Tuple[str, ...]:
         DESKTOP_SRC,
         IMPRESSION_SRC,
         CFR_SRC,
-        ECOSYSTEM_CLIENT_ID_HMAC_SRC,
-        ECOSYSTEM_CLIENT_ID_SRC,
         FXA_HMAC_SRC,
         FXA_SRC,
     ]
@@ -274,21 +261,6 @@ def fields(self) -> Tuple[str, ...]:
     user_id_target(
         table="firefox_accounts_derived.fxa_users_services_last_seen_v1"
     ): FXA_SRC,
-    # account ecosystem telemetry (AET)
-    DeleteTarget(
-        table="telemetry_stable.account_ecosystem_v4", field=DESKTOP_ECOSYSTEM_CLIENT_ID
-    ): ECOSYSTEM_CLIENT_ID_SRC,
-    DeleteTarget(
-        table="firefox_accounts_stable.account_ecosystem_v1", field=ECOSYSTEM_CLIENT_ID
-    ): ECOSYSTEM_CLIENT_ID_SRC,
-    DeleteTarget(
-        table="account_ecosystem_derived.ecosystem_client_id_lookup_v1",
-        field=ECOSYSTEM_CLIENT_ID_HASH,
-    ): ECOSYSTEM_CLIENT_ID_HMAC_SRC,
-    DeleteTarget(
-        table="account_ecosystem_derived.desktop_clients_daily_v1",
-        field=ECOSYSTEM_CLIENT_ID_HASH,
-    ): ECOSYSTEM_CLIENT_ID_HMAC_SRC,
     # legacy mobile
     DeleteTarget(
         table="telemetry_stable.core_v1",
@@ -369,9 +341,6 @@ def fields(self) -> Tuple[str, ...]:
         client_id_target(table="eng_workflow_stable.build_v1"),
         # other
         DeleteTarget(table="telemetry_stable.pioneer_study_v4", field=PIONEER_ID),
-        DeleteTarget(
-            table="telemetry_stable.pre_account_v4", field=DESKTOP_ECOSYSTEM_CLIENT_ID
-        ),
     ]
 }
 

dags.yaml

@@ -271,18 +271,6 @@ bqetl_fenix_event_rollup:
     retries: 2
     retry_delay: 30m
 
-bqetl_account_ecosystem:
-  schedule_interval: 0 2 * * *
-  description: >-
-    Tables related to the v1 implementation of Account Ecosystem Telemetry (AET)
-    which is currently on hold.
-  default_args:
-    owner: jklukas@mozilla.com
-    start_date: "2020-09-17"
-    email: ["jklukas@mozilla.com"]
-    retries: 2
-    retry_delay: 30m
-
 bqetl_stripe:
   schedule_interval: daily
   default_args: