More DE changes

Author: curtishoward

roles/common/defaults/main.yml

@@ -45,6 +45,7 @@ common__external_data_suffix:             "{{ globals.labels.external_data | def
 common__datalake_admin_suffix:            "{{ globals.labels.datalake_admin | default('dladmin') }}"
 common__ranger_audit_suffix:              "{{ globals.labels.ranger_audit | default('audit') }}"
 common__cml_suffix:                       "{{ globals.labels.cml | default('cml') }}"
+common__cde_suffix:                       "{{ globals.labels.cde | default('cde') }}"
 common__igw_suffix:                       "{{ globals.labels.internet_gateway | default('igw') }}"
 common__app_suffix:                       "{{ globals.labels.app | default('app') }}"
 common__group_suffix:                     "{{ globals.labels.group | default('group') }}"
@@ -141,4 +142,4 @@ common__include_datahub:                   "{{ datahub is defined | bool }}"
 common__include_opdb:                      "{{ opdb is defined | bool }}"
 
 # Teardown
-common__force_teardown:                    "{{ globals.force_teardown | default(False) }}"  # WARNING: This will purge your namespace and anything related to it, use with extreme caution
+common__force_teardown:                    "{{ globals.force_teardown | default(False) }}"  # WARNING: This will purge your namespace and anything related to it, use with extreme caution

roles/info/tasks/main.yml

@@ -34,7 +34,7 @@
         - name: Set fact for the artifacts directory path
           ansible.builtin.set_fact:
             __artifacts_directory_path: "{{ __artifacts_directory_create.path }}"
-      
+
     - name: Validate artifacts directory if it does exist
       when: __artifacts_directory.stat.exists
       block:
@@ -70,6 +70,11 @@
     env: "{{ info__env_name }}"
   register: __ml_info
 
+- name: Query CDP DE Services
+  cloudera.cloud.de_info:
+    env: "{{ info__env_name }}"
+  register: __de_info
+
 - name: Query CDP Operational DBs
   cloudera.cloud.opdb_info:
     env:  "{{ info__env_name }}"
@@ -82,11 +87,12 @@
       datalake: "{{ __datalake_info.datalakes | first | default({}) }}"
       datahubs: "{{ __datahubs_info.datahubs }}"
       workspaces: "{{ __ml_info.workspaces }}"
+      services: "{{ __de_info.services }}"
       operational_dbs: "{{ __opdb_info.databases }}"
 
 - name: Save the CDP deployment details locally
   when: info__create_deployment_details
   ansible.builtin.copy:
     content: "{{ { 'deployment': deployment } | to_nice_yaml(indent=2) }}"
     dest: "{{ __artifacts_directory_path | default('.') }}/deployment_info.yml"
-  delegate_to: localhost
+  delegate_to: localhost

roles/platform/tasks/initialize_teardown.yml

@@ -28,11 +28,12 @@
       - plat__env_info.environments[0].descendants.df | length == 0
       - plat__env_info.environments[0].descendants.dw | length == 0
       - plat__env_info.environments[0].descendants.opdb | length == 0
+      - plat__env_info.environments[0].descendants.de | length == 0
     fail_msg: "Environment {{ plat__env_name }} has one or more child services registered, please check and try again"
     quiet: yes
 
 - name: Include provider-specific initialization base
   ansible.builtin.include_tasks: "initialize_{{ plat__infra_type | lower }}.yml"
 
 - name: Include provider-specific resources for CDP Public Environment teardown
-  ansible.builtin.include_tasks: "initialize_teardown_{{ plat__infra_type | lower }}.yml"
+  ansible.builtin.include_tasks: "initialize_teardown_{{ plat__infra_type | lower }}.yml"

roles/platform/vars/main.yml

@@ -39,6 +39,8 @@ plat__cdp_iam_identities:
   dw_user: DWUser
   ml_admin: MLAdmin
   ml_user: MLUser
+  de_admin: DEAdmin
+  de_user: DEUser
 
 plat__cdp_iam_admin_group_resource_roles_default:
   - "{{ plat__cdp_iam_identities.env_admin }}"
@@ -47,11 +49,14 @@ plat__cdp_iam_admin_group_resource_roles_default:
   - "{{ plat__cdp_iam_identities.dw_user }}"
   - "{{ plat__cdp_iam_identities.ml_admin }}"
   - "{{ plat__cdp_iam_identities.ml_user }}"
+  - "{{ plat__cdp_iam_identities.de_admin }}"
+  - "{{ plat__cdp_iam_identities.de_user }}"
 
 plat__cdp_iam_user_group_resource_roles_default:
   - "{{ plat__cdp_iam_identities.env_user }}"
   - "{{ plat__cdp_iam_identities.dw_user }}"
   - "{{ plat__cdp_iam_identities.ml_user }}"
+  - "{{ plat__cdp_iam_identities.de_user }}"
 
 plat__gcp_xaccount_policy_bindings_default:
   - "roles/compute.instanceAdmin.v1"
@@ -73,4 +78,4 @@ plat__azure_roles:
   storown: 'Storage Blob Data Owner'
   storcnt: 'Storage Blob Data Contributor'
   contrib: 'Contributor'
-  owner: 'Owner'
+  owner: 'Owner'

roles/runtime/defaults/main.yml

@@ -64,13 +64,15 @@ run__ml_force_delete:               "{{ ml.force_delete | default (run__force_te
 run__ml_remove_storage:             "{{ ml.remove_storage | default (run__force_teardown) }}"
 run__ml_public_loadbalancer:        "{{ ml.public_loadbalancer | default(run__public_endpoint_access) }}"
 
+run__de_definitions:                "{{ de.definitions | default([{}]) }}"
+run__de_suffix:                     "{{ de.suffix | default('de') }}"
+run__de_tags:                       "{{ de.tags | default(common__tags) }}"
+run__de_force_delete:               "{{ de.force_delete | default (run__force_teardown) }}"
+
 run__dw_definitions:                "{{ dw.definitions | default([{}]) }}"
 run__dw_suffix:                     "{{ dw.suffix | default('dw') }}"
 run__dw_force_delete:               "{{ dw.force_delete | default (run__force_teardown) }}"
 
-run__de_definitions:                "{{ de.definitions | default([{}]) }}"
-run__de_suffix:                     "{{ de.suffix | default('de') }}"
-
 run__df_nodes_min:                  "{{ df.min_k8s_nodes | default(3) }}"
 run__df_nodes_max:                  "{{ df.max_k8s_nodes | default(5) }}"
 run__df_public_loadbalancer:        "{{ df.public_loadbalancer | default(run__public_endpoint_access) }}"

roles/runtime/tasks/initialize_base.yml

@@ -162,18 +162,19 @@
         loop_var: __ml_config
         label: "{{ config.name }}"
 
-- name: Prepare for CDP DE experiences
+- name: Prepare for CDP DE Service experiences
   when: run__include_de
   block:
-    - name: Retrieve CDP DE experiences
-      cloudera.cloud.de_info:
-        env: "{{ run__env_name }}"
-      register: run__de_list
-
-- name: Prepare for CDP DW experiences
-  when: run__include_dw
-  block:
-    - name: Retrieve CDP DW experiences
-      cloudera.cloud.dw_cluster_info:
-        env: "{{ run__env_name }}"
-      register: run__dw_list
+    - name: Construct CDP DE Service configurations
+      ansible.builtin.set_fact:
+        run__de_configs: "{{ run__de_configs | default([]) | union([config]) }}"
+      vars:
+        include: "{{ lookup('template', __de_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
+        overlay_tags: "{{ __de_config.tags | default(include.tags) | default(run__de_tags) }}"
+        config:
+          name: "{{ __de_config.name | default([run__namespace_cdp, __de_config.suffix | default(include.suffix) | default(run__de_suffix)] | join('-')) }}"
+          raw: "{{ __de_config }}"
+      loop: "{{ run__de_definitions }}"
+      loop_control:
+        loop_var: __de_config
+        label: "{{ config.name }}"

roles/runtime/tasks/initialize_teardown.yml

@@ -68,4 +68,16 @@
       loop: "{{ run__env_info.environments[0].descendants.ml }}"
       loop_control:
         loop_var: __ml_config
-        label: "{{ __ml_config.instanceName }}"
+        label: "{{ __ml_config.instanceName }}"
+
+    - name: Prepare teardown list of all DE Services in Environment
+      ansible.builtin.set_fact:
+        run__de_configs: "{{ run__de_configs | default([]) | union([config]) }}"
+      vars:
+        config:
+          name: "{{ __de_config.name }}"
+          env: "{{ __de_config.environmentName }}"
+      loop: "{{ run__env_info.environments[0].descendants.de }}"
+      loop_control:
+        loop_var: __de_config
+        label: "{{ __de_config.name }}"

roles/runtime/tasks/setup_base.yml

@@ -59,6 +59,34 @@
   poll: 0
   register: __ml_builds
 
+- name: Execute CDP DE Service experiences setup
+  when: run__include_de
+  cloudera.cloud.de:
+    name: "{{ __de_config_item.name }}"
+    env: "{{ run__env_name }}"
+    instance_type: "{{ __de_config_item.raw.instance_type }}"
+    minimum_instances: "{{ __de_config_item.raw.minimum_instances }}"
+    maximum_instances: "{{ __de_config_item.raw.maximum_instances }}"
+    minimum_spot_instances: "{{ __de_config_item.raw.minimum_spot_instances | default(omit) }}"
+    maximum_spot_instances: "{{ __de_config_item.raw.maximum_spot_instances | default(omit) }}"
+    chart_value_overrides: "{{ __de_config_item.raw.chart_value_overrides | default(omit) }}"
+    enable_public_endpoint: "{{ __de_config_item.enable_public_endpoint | default(omit) }}"
+    enable_workload_analytics: "{{ __de_config_item.enable_workload_analytics | default(omit) }}"
+    initial_instances: "{{ __de_config_item.initial_instances | default(omit) }}"
+    initial_spot_instances: "{{ __de_config_item.initial_spot_instances | default(omit) }}"
+    root_volume_size: "{{ __de_config_item.root_volume_size | default(omit) }}"
+    skip_validation: "{{ __de_config_item.skip_validation | default(omit) }}"
+    tags: "{{ __de_config_item.raw.tags | default(omit) }}"
+    use_ssd: "{{ __de_config_item.use_ssd | default(omit) }}"
+    whitelist_ips: "{{ __de_config_item.raw.whitelist_ips | default(omit) }}"
+  loop_control:
+    loop_var: __de_config_item
+    label: "{{ __de_config_item.name | default ('None') }}"
+  loop: "{{ run__de_configs }}"
+  async: 7200 # 2 hour timeout
+  poll: 0
+  register: __de_builds
+
 - name: Execute CDP OpDB setup
   when: run__include_opdb
   cloudera.cloud.opdb:
@@ -85,19 +113,6 @@
     state: present
     wait: no
 
-- name: Execute CDP DE Workspace experiences setup
-  when: run__include_de
-  cloudera.cloud.de:
-    name: "{{ __de_config_item.name }}"
-    env: "{{ run__env_name }}"
-  loop_control:
-    loop_var: __de_config_item
-    label: "{{ __de_config_item.name | default ('None') }}"
-  loop: "{{ run__de_configs }}"
-  async: 3600 # 1 hour timeout
-  poll: 0
-  register: __de_builds
-
 - name: Wait for CDP Datahub deployments to complete
   when: run__include_datahub
   ansible.builtin.async_status:
@@ -124,6 +139,19 @@
   retries: 120
   delay: 30
 
+- name: Wait for CDP DE Service experiences to complete
+  when: run__include_de
+  ansible.builtin.async_status:
+    jid: "{{ __de_build.ansible_job_id }}"
+  loop_control:
+    loop_var: __de_build
+    label: "{{ __de_build.__de_config_item.name | default ('None') }}"
+  loop: "{{ __de_builds.results }}"
+  register: __de_builds_async
+  until: __de_builds_async.finished
+  retries: 120
+  delay: 60
+
 - name: Wait for CDP OpDB deployments to complete
   when: run__include_opdb
   ansible.builtin.async_status:
@@ -142,16 +170,3 @@
   cloudera.cloud.df:
     name: "{{ run__cdp_env_crn }}"
     wait: yes
-
-- name: Wait for CDP DE Workspace experiences to complete
-  when: run__include_de
-  ansible.builtin.async_status:
-    jid: "{{ __de_build.ansible_job_id }}"
-  loop_control:
-    loop_var: __de_build
-    label: "{{ __de_build.__de_config_item.name | default ('None') }}"
-  loop: "{{ __de_builds.results }}"
-  register: __de_builds_async
-  until: __de_builds_async.finished
-  retries: 120
-  delay: 30

roles/runtime/tasks/teardown_base.yml

@@ -79,6 +79,27 @@
   poll: 0
   register: __ml_teardowns_info
 
+- name: Execute CDP DE Service teardown
+  when:
+    - run__include_de or run__force_teardown | bool
+    - run__de_configs is defined
+    - run__de_configs | length > 0
+  cloudera.cloud.de:
+    name: "{{ __de_config_item.name }}"
+    env: "{{ run__env_name }}"
+    instance_type: "{{ __de_config_item.raw.instance_type }}"
+    minimum_instances: "{{ __de_config_item.raw.minimum_instances }}"
+    maximum_instances: "{{ __de_config_item.raw.maximum_instances }}"
+    state: absent
+    force_delete: "{{ run__de_force_delete }}"
+  loop_control:
+    loop_var: __de_config_item
+    label: "{{ __de_config_item.name | default('de') }}"
+  loop: "{{ run__de_configs }}"
+  async: 3600 # 1 hour timeout
+  poll: 0
+  register: __de_teardowns_info
+
 - name: Execute CDP Datahub teardown
   when:
     - run__include_datahub or run__force_teardown | bool
@@ -113,6 +134,22 @@
   retries: 120
   delay: 30
 
+- name: Wait for CDP DE Service deployments to decommission
+  when:
+    - __de_teardowns_info is defined
+    - __de_teardowns_info.results is defined
+    - __de_teardowns_info.results | length > 0
+  ansible.builtin.async_status:
+    jid: "{{ __de_teardown.ansible_job_id }}"
+  loop_control:
+    loop_var: __de_teardown
+    label: "{{ __de_teardown.__de_config_item.name | default('de') }}"
+  loop: "{{ __de_teardowns_info.results }}"
+  register: __de_teardowns_async
+  until: __de_teardowns_async.finished
+  retries: 120
+  delay: 30
+
 - name: Wait for CDP Datahub deployments to decommission
   when:
     - __datahub_teardowns_info is defined