Merge pull request #103 from TechnologyEnhancedLearning/staging-data-test-team-shared

Staging data test team shared

Author: Phil-NHS

databricks.yml

@@ -258,7 +258,11 @@ targets:
         env_analysis_batch_id: ${var.serverless_usage_policies.prod_analysis_batch_id}
         env_lh_live_id: ${var.serverless_usage_policies.prod_lh_live_id}
         # Note: You can leave test IDs blank here or map them to a default 
-        # since Prod doesn't have test policies.
+        # since Prod doesn't have test policies. However frustratingly despite excluding resource/test the bundle validation check it anyway
+        env_test_all_id: "NA"
+        env_test_dq_id: "NA"
+        env_test_int_id: "NA"
+        env_test_unit_id: "NA"
     permissions:
       - group_name: prod_env_users
         level: CAN_VIEW

docs/Cost Visibility.md

@@ -41,6 +41,8 @@ qqqq maybe owner names and groups need to become the same. YES
 
 Unfortunately currently dab is not setting permissions for sps to use the serverless policys so they have to be set through the UI
 
+*Maybe we should have tags in Staging_Analysis_Batch for ingestion or medallion, but until we have some good queries and dashboards the most useful tags to have wont be clear*
+
 | Policy Name | Env | CostCentre | Test / Owner | 
 |---|---|---|---|
 |-------Numbered / General Policies-------| |||

resources/jobs/utils/full_data_refresh.yml

@@ -7,7 +7,9 @@ resources:
         - task_key: run_bronze_pipeline
           pipeline_task:
             pipeline_id: ${resources.pipelines.pipeline_ods_ingestion.id}
-            
+      # not included serverless policy as this gets run by tests for the DQ tests, the best approach would be policy per task in the job calling this, and seperate the notebooks so each test task can be seperate [Not sure can do it task based](https://learn.microsoft.com/en-us/azure/databricks/admin/usage/budget-policies?source=recommendations#assign-permissions-on-a-policy)
+
+
         # Exclude streaming if they are triggered whilst active an error occurs
         # - task_key: run_silver_pipeline
         #   depends_on:

resources/pipeline/ingest/ods_ingestion.yml

@@ -27,6 +27,7 @@ resources:
   pipelines:
     pipeline_ods_ingestion:
       name: ods_ingestion
+      usage_policy_id: ${var.env_serverless_usage_policies.env_analysis_batch_id}
       libraries:
         - glob:
             include: ../../../src/ingestion/ods_ingest.py

resources/pipeline/silver/isreporter_dlt.yml

@@ -3,6 +3,7 @@ resources:
   pipelines:
     pipeline_isreporter_live_dlt:
       name: isreporter_live_dlt
+      usage_policy_id: ${var.env_serverless_usage_policies.env_lh_live_id}
       libraries:
         - glob:      # go up resources/pipelines/silver then down
             include: ../../../src/silver/isreporter_dlt.py