Add support for GCP backup storage bucket (#172)

* Add support for specifying the GCP backups storage bucket
* Update backup_location parameter name for GCP env
* Remove duplicate plat__data_path variable
* Update configuration parameter docs

Signed-off-by: Jesus Perez Rey <jprey@bluemetrix.com>
Signed-off-by: Jim Enright <jenright@cloudera.com>
Co-authored-by: Jesus Perez Rey <jprey@bluemetrix.com>

Author: jimright

docs/configuration.yml

@@ -268,6 +268,7 @@ env:
       suffix:
     storage:
       path:
+        backups:
         data:
         logs:
       suffix:
@@ -423,6 +424,7 @@ infra:
     region:
     storage:
       path:
+        backups:
         data:
         logs:
   security_group:
@@ -438,6 +440,7 @@ infra:
   storage:
     name:
     path:
+      backups:
       data:
       de:
       logs:

roles/common/defaults/main.yml

@@ -104,6 +104,7 @@ common__ml_path:                          "{{ infra.storage.path.ml | default('d
 common__de_path:                          "{{ infra.storage.path.de | default('dataeng') }}"
 common__logs_path:                        "{{ infra.storage.path.logs | default(common__logs_suffix) }}"
 common__data_path:                        "{{ infra.storage.path.data | default(common__data_suffix) }}"
+common__backups_path:                     "{{ infra.storage.path.backups | default(common__logs_path) }}"
 common__ranger_audit_path:                "{{ infra.storage.path.ranger_audit | default('ranger/audit') }}"
 
 # AWS Infra

roles/infrastructure/defaults/main.yml

@@ -68,6 +68,7 @@ infra__storage_name:                "{{ common__storage_name }}"
 
 infra__logs_path:                   "{{ common__logs_path }}"
 infra__data_path:                   "{{ common__data_path }}"
+infra__backups_path:                "{{ common__backups_path }}"
 infra__ranger_audit_path:           "{{ common__ranger_audit_path }}"
 
 infra__public_key_path:             "{{ globals.ssh.key_path | default('~/.ssh') }}"
@@ -135,8 +136,9 @@ infra__aws_private_endpoints:       "{{ infra.aws.vpc.private_endpoints | defaul
 # GCP
 infra__gcp_project:                 "{{ common__gcp_project }}"
 
-infra__gcp_storage_location_data:   "{{ infra.gcp.storage.path.data | default([infra__storage_name, infra__data_path] | join('-')) }}"
-infra__gcp_storage_location_logs:   "{{ infra.gcp.storage.path.logs | default([infra__storage_name, infra__logs_path] | join('-')) }}"
+infra__gcp_storage_location_data:    "{{ infra.gcp.storage.path.data | default([infra__storage_name, infra__data_path] | join('-')) }}"
+infra__gcp_storage_location_logs:    "{{ infra.gcp.storage.path.logs | default([infra__storage_name, infra__logs_path] | join('-')) }}"
+infra__gcp_storage_location_backups: "{{ infra.gcp.storage.path.backups | default(infra__gcp_storage_location_logs) }}"
 
 infra__gcp_cloud_router_name_suffix:  "{{ infra.gcp.network.router.name_suffix | default('router') }}"
 infra__gcp_cloud_router_name:         "{{ infra.gcp.network.router.name | default([infra__namespace, infra__gcp_cloud_router_name_suffix] | join('-')) }}"
@@ -175,4 +177,4 @@ infra__utlity_bucket_name:            "{{ globals.utility_bucket_name | default(
 
 # Teardown
 infra__force_teardown:              "{{ common__force_teardown }}"
-infra__env_name:                    "{{ common__env_name }}"  # Used for purge lookups
+infra__env_name:                    "{{ common__env_name }}"  # Used for purge lookups

roles/infrastructure/tasks/initialize_gcp.yml

@@ -38,11 +38,12 @@
     fail_msg: "Gcloud Collection failed to retrieve resources, you may need to run 'gcloud auth login' or 'gcloud init': {{ __gcp_vpc_info }}"
     quiet: yes
 
-- name: Set fact for Log and Data locations
+- name: Set fact for Log, Backups and Data locations
   ansible.builtin.set_fact:
     infra__gcp_storage_locations: "{{ infra__gcp_storage_locations | default([]) | union([__gcp_storage_location_item]) }}"
   loop_control:
     loop_var: __gcp_storage_location_item
   loop:
     - "{{ infra__gcp_storage_location_data }}"
-    - "{{ infra__gcp_storage_location_logs }}"
+    - "{{ infra__gcp_storage_location_logs }}"
+    - "{{ infra__gcp_storage_location_backups }}"

roles/platform/defaults/main.yml

@@ -52,6 +52,7 @@ plat__storage_name:                           "{{ common__storage_name }}"
 
 plat__logs_path:                              "{{ common__logs_path }}"
 plat__data_path:                              "{{ common__data_path }}"
+plat__backup_path:                            "{{ common__backups_path }}"
 
 plat__public_key_id:                          "{{ common__public_key_id }}"
 plat__public_key_text:                        "{{ common__public_key_text }}"
@@ -168,6 +169,7 @@ plat__gcp_idbroker_identity_name:             "{{ env.gcp.role.name.idbroker | d
 
 plat__gcp_storage_location_data:              "{{ env.gcp.storage.path.data | default([plat__storage_name, plat__data_path] | join('-')) }}"
 plat__gcp_storage_location_logs:              "{{ env.gcp.storage.path.logs | default([plat__storage_name, plat__logs_path] | join('-')) }}"
+plat__gcp_storage_location_backups:           "{{ env.gcp.storage.path.backups | default(plat__gcp_storage_location_logs) }}"
 
 plat__gcp_xaccount_policy_bindings:           "{{ env.gcp.bindings.cross_account | default(plat__gcp_xaccount_policy_bindings_default) }}"
 plat__gcp_log_role_perms:                     "{{ env.gcp.bindings.logs | default(plat__gcp_log_policy_bindings_default) }}"

roles/platform/tasks/setup_gcp_authz.yml

@@ -167,6 +167,8 @@
       loop:
         - account: "serviceAccount:{{ plat__gcp_log_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin"
           bucket: "{{ plat__gcp_storage_location_logs }}"
+        - account: "serviceAccount:{{ plat__gcp_log_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin"
+          bucket: "{{ plat__gcp_storage_location_backups }}"
         - account: "serviceAccount:{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin"
           bucket: "{{ plat__gcp_storage_location_data }}"
         - account: "serviceAccount:{{ plat__gcp_ranger_audit_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin"

roles/platform/tasks/setup_gcp_env.yml

@@ -25,6 +25,7 @@
     public_ip: "{{ plat__use_public_ip }}"
     log_location: "gs://{{ plat__gcp_storage_location_logs }}"
     log_identity: "{{ plat__gcp_log_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
+    backup_location: "gs://{{ plat__gcp_storage_location_backups }}"
     vpc_id: "{{ plat__vpc_name }}"
     subnet_ids:
       - "{{ plat__gcp_subnet_id if plat__gcp_subnet_id else plat__gcp_subnets_discovered[0].name }}"  # TODO - Check in validation_gcp.yml -- CDP on GCP only supports a single subnet deployment