nhp_data/databricks_workflows/nhp_data-reference_data.yaml at main · The-Strategy-Unit/nhp_data · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
resources:
  jobs:
    Generate_NHP_Data_Reference_Data:
      name: Generate NHP Data (Reference Data)
      tasks:
        - task_key: run_reference_data
          condition_task:
            op: EQUAL_TO
            left: "{{job.parameters.run_reference_data}}"
            right: "True"
        - task_key: ods_trusts
          depends_on:
            - task_key: run_reference_data
              outcome: "true"
          python_wheel_task:
            package_name: nhp_data
            entry_point: reference-ods_trusts
          job_cluster_key: nhp_data
          libraries:
            - whl: ../dist/*.whl
        - task_key: lsoa_lookups
          depends_on:
            - task_key: ods_trusts
          python_wheel_task:
            package_name: nhp_data
            entry_point: reference-lsoa_lookups
          job_cluster_key: nhp_data
          libraries:
            - whl: ../dist/*.whl
        - task_key: population_by_lsoa21
          depends_on:
            - task_key: lsoa_lookups
          python_wheel_task:
            package_name: nhp_data
            entry_point: reference-population_by_lsoa21
          job_cluster_key: nhp_data
          libraries:
            - whl: ../dist/*.whl
        - task_key: provider_main_icb
          depends_on:
            - task_key: population_by_lsoa21
          python_wheel_task:
            package_name: nhp_data
            entry_point: reference-provider_main_icb
          job_cluster_key: nhp_data
          libraries:
            - whl: ../dist/*.whl
        - task_key: population_by_imd_decile
          depends_on:
            - task_key: provider_main_icb
          python_wheel_task:
            package_name: nhp_data
            entry_point: reference-population_by_imd_decile
            parameters:
              - "{{job.parameters.population_by_imd_decile_base_year}}"
          job_cluster_key: nhp_data
          libraries:
            - whl: ../dist/*.whl
        - task_key: create_provider_catchments
          depends_on:
            - task_key: population_by_imd_decile
          python_wheel_task:
            package_name: nhp_data
            entry_point: reference-provider_catchments
          job_cluster_key: nhp_data
          libraries:
            - whl: ../dist/*.whl
        - task_key: create_icb_catchments
          depends_on:
            - task_key: create_provider_catchments
          python_wheel_task:
            package_name: nhp_data
            entry_point: reference-icb_catchments
          job_cluster_key: nhp_data
          libraries:
            - whl: ../dist/*.whl
        - task_key: create_day_procedure_code_list
          depends_on:
            - task_key: create_icb_catchments
          python_wheel_task:
            package_name: nhp_data
            entry_point: reference-day_procedures
            parameters:
              - "{{job.parameters.day_procedures_base_year}}"
          job_cluster_key: nhp_data
      job_clusters:
        - job_cluster_key: nhp_data
          new_cluster:
            spark_version: 17.3.x-scala2.13
            spark_conf:
              spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite: "true"
              spark.databricks.delta.properties.defaults.autoOptimize.autoCompact: "true"
              spark.databricks.sql.initial.catalog.namespace: udal_lake_mart
            azure_attributes:
              availability: ON_DEMAND_AZURE
              spot_bid_max_price: 100
            node_type_id: Standard_D4ds_v5
            custom_tags:
              Application: UDAL
              Workload_Type: Job
              Workload_Size: Small
            spark_env_vars:
              PYSPARK_PYTHON: /databricks/python3/bin/python3
            policy_id: 000816B2986C0B29
            data_security_mode: USER_ISOLATION
            runtime_engine: STANDARD
            kind: CLASSIC_PREVIEW
            is_single_node: false
            autoscale:
              min_workers: 1
              max_workers: 4
      tags:
        group: nhp_data
        env: ${bundle.target}
      queue:
        enabled: true
      parameters:
        - name: run_reference_data
          default: "True"
        - name: day_procedures_base_year
          default: "202324"
        - name: population_by_imd_decile_base_year
          default: "202324"