Rebase of devel-pvc-update onto devel

requirements_aws.txt

@@ -14,7 +14,7 @@
 
 # Amazon Web Services
 
-awscli>=1.18.222
+awscli>=1.25.22
 boto>=2.49.0
-botocore>=1.12.249
-boto3>=1.9.249
+botocore>=1.27.22
+boto3>=1.24.22

requirements_azure.txt

@@ -14,8 +14,8 @@
 
 # Azure
 
-# NOTE: The Azure CLI is installed as a OS package.
+# NOTE: The Azure CLI is installed as an OS package.
 
 # Azure Collection requirements, which are distinct from Azure CLI
 -r https://raw.githubusercontent.com/ansible-collections/azure/dev/requirements-azure.txt
-azure-mgmt-netapp>=1.0.0    # NetApp Collection requirements
+azure-mgmt-netapp>=8.0.0    # NetApp Collection requirements

requirements_gcp.txt

@@ -15,4 +15,4 @@
 # Google Cloud
 
 # NOTE: google-cloud-sdk, including the CLI, is installed as OS package.
-google-auth
+google-auth>=2.9.0

roles/auto_repo_mirror/defaults/main.yml

@@ -0,0 +1,3 @@
+default_enable_auto_repo_mirror: no
+default_download_link_expiry: 3600
+default_auto_repo_mirror_prefix: cache

roles/auto_repo_mirror/tasks/inject.yml

@@ -0,0 +1,139 @@
+---
+
+# Copyright 2021 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Check if Download Mirror cache file exists
+  register: __auto_repo_mirror_file_stat
+  ansible.builtin.stat:
+    path: "{{ init__auto_repo_mirror_artefact }}"
+
+- name: Prepare for Download Mirror parsing
+  when:
+    - __auto_repo_mirror_file_stat.stat.exists
+    - use_auto_repo_mirror | default(default_enable_auto_repo_mirror) | bool
+  block:
+    - name: Check a custom_repo is part of the cluster definition if using Download Mirror
+      when: use_auto_repo_mirror | default(default_enable_auto_repo_mirror) | bool
+      ansible.builtin.assert:
+        that: "'custom_repo' in groups"
+        fail_msg: "You must have a custom_repo in your Cluster Inventory when using Download Mirror"
+
+    - name: Handle AWS cache key generation
+      when: globals.infra_type == 'aws'
+      block:
+        - name: Get AWS Account Info
+          amazon.aws.aws_caller_info:
+          register: __aws_caller_info
+
+        - name: Set parcel cache ini lookup key
+          ansible.builtin.set_fact:
+            init__auto_repo_mirror_ini_key: "{{ __aws_caller_info.account }}"
+
+# Ini lookup fails when section is not present, but lacks good control characteristics that I can find
+- name: Determine if there is a relevant cache entry
+  when:
+    - init__auto_repo_mirror_ini_key is defined
+    - use_auto_repo_mirror | default(default_enable_auto_repo_mirror) | bool
+  ignore_errors: yes
+  ansible.builtin.set_fact:
+    __auto_repo_mirror_ini_entry: "{{ lookup('ini', __ini_lookup) }}"
+  vars:
+    __ini_lookup: ".+{{ init__auto_repo_mirror_ini_key }}.+ section={{ globals.infra_type }}:{{ globals.region }} file={{ init__auto_repo_mirror_artefact }} re=yes"
+
+- name: Handle Download Mirror injection if cache entry found
+  when:
+    - __auto_repo_mirror_ini_entry is defined
+  block:
+    - name: Generate a unique name
+      set_fact:
+        __tmp_cluster_file: "{{ ['/tmp', 99999999 | random | to_uuid] | path_join }}"
+
+    - name: Copy Cluster definition to temp file
+      copy:
+        src: "{{ init__cluster_definition_file }}"
+        dest: "{{ __tmp_cluster_file }}"
+
+    - name: Inject Parcel cache entries to Repository URLs
+      ansible.builtin.replace:
+        name: "{{ __tmp_cluster_file }}"
+        regexp: '^(\s+\-\s)https://archive\.cloudera\.com(\/.+)$'
+        replace: '\1http://{{ groups.custom_repo | first }}\2'
+
+    - name: Set Cluster Definition file to Temp file with parcel cache entries
+      ansible.builtin.set_fact:
+        init__cluster_definition_file: "{{ __tmp_cluster_file }}"
+
+    - name: Fetch repositories from cluster definition
+      ansible.builtin.set_fact:
+        init__preparse_repo_listing: "{{ lookup('file', init__cluster_definition_file ) | from_yaml | json_query('clusters[*].repositories') | flatten }}"
+
+    - name: Prepare lookup list of Repository entries
+      loop: "{{ init__preparse_repo_listing }}"
+      loop_control:
+        loop_var: __cluster_repo_item
+      ansible.builtin.set_fact:
+        init__cluster_repo_entries: "{{ init__cluster_repo_entries | default([]) + [__cluster_repo_item | urlsplit('path') ] }}"
+
+    - name: Create list of Download Mirror URLs filtered to required repositories and distros
+      loop: "{{ init__cluster_repo_entries }}"
+      loop_control:
+        loop_var: __cluster_repo_path_item
+      ansible.builtin.set_fact:
+        init__urls_to_sign: "{{ init__urls_to_sign
+          | default([]) + __auto_repo_mirror_ini_entry
+          | select('search', __cluster_repo_path_item)
+          | select('search', init__parcel_distro)
+          | list }}"
+
+    - name: Include Cloudera Manager Tarball
+      ansible.builtin.set_fact:
+        init__urls_to_sign: "{{ init__urls_to_sign
+            | default([]) + __auto_repo_mirror_ini_entry
+            | select('search', 'repo-as-tarball')
+            | select('search', cloudera_manager_version)
+            | select('search', cm_distro_select[init__parcel_distro]['version'] | string + '.tar')
+            | list }}"
+
+# TODO: Filter to relevent version manifests, not all manifests, just to be tidy
+    - name: Ensure manifest is included in Download Mirror URLs if present
+      loop: "{{ init__cluster_repo_entries }}"
+      loop_control:
+        loop_var: __cluster_repo_path_item
+      ansible.builtin.set_fact:
+        init__urls_to_sign: "{{ init__urls_to_sign
+            | default([]) + __auto_repo_mirror_ini_entry
+            | select('search', 'manifest.json')
+            | list }}"
+
+- name: Get AWS Specific download URIs
+  when:
+    - globals.infra_type == 'aws'
+    - init__urls_to_sign is defined
+  block:
+    - name: Generate signed URIs for hosted parcels to be pulled into custom_repo
+      register: __s3_signed_uris
+      loop: "{{ init__urls_to_sign }}"
+      loop_control:
+        loop_var: __s3_bucket_uri
+      amazon.aws.aws_s3:
+        bucket: "{{ __s3_bucket_uri | regex_replace('^.+//(.+)\\.s3.+$', '\\1') }}"
+        object: "{{ __s3_bucket_uri | regex_replace('^.+amazonaws\\.com(.+)$', '\\1') }}"
+        ignore_nonexistent_bucket: yes
+        expiry: "{{ download_link_expiry | default(default_download_link_expiry) }}"
+        mode: geturl
+
+    - name: Set List of files to download to custom_repo
+      ansible.builtin.set_fact:
+        auto_repo_mirror_file_list: "{{ __s3_signed_uris.results | json_query('[*].url') | list  }}"

roles/auto_repo_mirror/tasks/parse_definition_for_mirror_targets.yml

@@ -0,0 +1,123 @@
+---
+
+# Copyright 2021 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Read in cluster definition without jinja parsing=
+- name: Fetch repositories from cluster definition
+  ansible.builtin.set_fact:
+    init__preparse_repo_listing: "{{ lookup('file', init__cluster_definition_file ) | from_yaml | json_query('clusters[*].repositories') | flatten }}"
+
+- name: Check that a Cloudera License is presented if mirroring from files behind Cloudera Subscription
+  when: init__preparse_repo_listing | select('search', 'archive.cloudera.com/p') | length > 0
+  ansible.builtin.assert:
+    that:
+      - globals.cloudera_license_file is defined
+      - globals.cloudera_license_file | length > 0
+    fail_msg: "You must Supply a Cloudera License file to download and mirror files from archive.cloudera.com"
+
+- name: Separate direct Repos from tarballs and set Initial Facts
+  ansible.builtin.set_fact:
+    __init_parcel_repos: "{{ init__preparse_repo_listing | reject('search', 'tar.gz') | default([]) }}"
+    __init_tarball_links: "{{ init__preparse_repo_listing | select('search', 'tar.gz') | default([]) }}"
+
+# This sets 'manifests' on the calling host, and provides the repo login details
+- name: Get Parcel Manifests
+  ansible.builtin.include_role:
+    name: cloudera.cluster.deployment.repometa
+    public: yes
+  vars:
+    repositories: "{{ __init_parcel_repos }}"
+    cluster_os_distribution: "{{ init__parcel_distro }}"
+
+- name: Extract Parcel URLs from Manifests
+  ansible.builtin.set_fact:
+    __parcel_urls: "{{ manifests.results | cloudera.cluster.extract_parcel_urls }}"
+    __parcel_distro_search_term: "{{ init__parcel_distro }}.parcel"
+
+- name: Filter Parcels by distro
+  ansible.builtin.set_fact:
+    __filtered_parcel_urls: "{{ __parcel_urls | select('search', __parcel_distro_search_term ) | list }}"
+
+- name: Prepare target Download Mirror listing with parcels and attendant files
+  when: __filtered_parcel_urls | length > 0
+  loop: "{{ __filtered_parcel_urls }}"
+  loop_control:
+    loop_var: __filtered_parcel_item
+  ansible.builtin.set_fact:
+    init__file_mirror_targets: "{{ init__file_mirror_targets | default(__init_tarball_links) + [__filtered_parcel_item, __filtered_parcel_item + '.sha1', __filtered_parcel_item + '.sha', __filtered_parcel_item + '.sha256', __filtered_parcel_item.replace(__filtered_parcel_item | basename, 'manifest.json') ] }}"
+
+# Explicitly set version from parcel distro as Ansible controller could be different OS from target cluster
+- name: Determine Cloudera-Manager Distro and Version
+  ansible.builtin.set_fact:
+    init__cloudera_manager_distro_name: "{{ cm_distro_select[init__parcel_distro]['name'] }}"
+    init__cloudera_manager_distro_version: "{{ cm_distro_select[init__parcel_distro]['version'] }}"
+
+# This sets 'cloudera_manager_repo_url' on the calling host
+- name: Determine Cloudera Manager Repo
+  ansible.builtin.include_role:
+    role: cloudera.cluster.cloudera_manager.repo
+  vars:
+    install_repo_on_host: no
+    clusters: []
+    cloudera_manager_distro_name: "{{ init__cloudera_manager_distro_name }}"
+    cloudera_manager_distro_version: "{{ init__cloudera_manager_distro_version }}"
+
+- name: Add Cloudera Manager Repo to File Mirror list
+  ansible.builtin.set_fact:
+    init__file_mirror_targets: "{{ init__file_mirror_targets + [cloudera_manager_repo_url | regex_replace('^(.+\\/(\\d\\.\\d\\.\\d)\\/)(\\w+)\\/.+$', '\\1' + 'repo-as-tarball/cm' + '\\2' + '-' + '\\3' + '.tar.gz')] }}"
+
+- name: Include CSDs if set
+  when:
+    - cloudera_manager_csds is defined
+    - cloudera_manager_csds | length > 0
+  ansible.builtin.set_fact:
+    init__file_mirror_targets: "{{ init__file_mirror_targets + cloudera_manager_csds }}"
+
+- name: Resolve Download Mirror for AWS
+  when: globals.infra_type == 'aws'
+  block:
+    - name: Get AWS Account Info
+      amazon.aws.aws_caller_info:
+      register: __aws_caller_info
+
+    - name: Prepare Localised Download Mirror utility bucket name
+      ansible.builtin.set_fact:
+        init__auto_repo_mirror_bucket_name: "{{ utility_bucket_name | default([ auto_repo_mirror_prefix | default(default_auto_repo_mirror_prefix), __aws_caller_info.account, globals.region ] | join('-') ) }}"
+
+    - name: List current target cache contents if any exist
+      register: __auto_repo_mirror_lookup_initial
+      failed_when:
+        - __auto_repo_mirror_lookup_initial.s3_keys is not defined
+        - "'cannot be found' not in __auto_repo_mirror_lookup_initial.msg"
+      amazon.aws.aws_s3:
+        bucket: "{{ init__auto_repo_mirror_bucket_name }}"
+        mode: list
+
+    - name: Filter Files not already in mirror to be downloaded
+      when: __auto_repo_mirror_lookup_initial.s3_keys is defined
+      loop: "{{ __auto_repo_mirror_lookup_initial.s3_keys }}"
+      loop_control:
+        loop_var: __init_s3key_item
+      ansible.builtin.set_fact:
+        init__file_mirror_targets: "{{ init__file_mirror_targets | reject('match', '^.+' + __init_s3key_item + '$') | list }}"
+
+- name: Set Download Mirror details in Globals
+  ansible.builtin.set_fact:
+    globals: "{{ globals | default({}) | combine( __auto_repo_mirror_spec, recursive=True ) }}"
+  vars:
+    __auto_repo_mirror_spec:
+      auto_repo_mirror_targets: "{{ init__file_mirror_targets }}"
+      utility_bucket_name: "{{ init__auto_repo_mirror_bucket_name }}"
+      create_utility_service: "{{ init__file_mirror_targets | length > 0 | bool }}"