Merge branch 'master' of https://github.com/kubernetes-sigs/kubespray

* 'master' of https://github.com/kubernetes-sigs/kubespray:
  Patch Calico for V3.14.0 missing CR and CRD (kubernetes-sigs#6276)
  Explicitly set ETCDCTL_API and use ETCDCTL_ENDPOINTS (kubernetes-sigs#6327)
  Add additional metadata configuration options to external Openstack CCM (kubernetes-sigs#6338) (kubernetes-sigs#6339)

docs/openstack.md

@@ -108,5 +108,11 @@ The new cloud provider is configured to have Octavia by default in Kubespray.
   - ""
   ```
 
+- You can override the default OpenStack metadata configuration (see [#6338](https://github.com/kubernetes-sigs/kubespray/issues/6338) for explanation):
+
+  ```yaml
+  external_openstack_metadata_search_order: "configDrive,metadataService"
+  ```
+
 - Run `source path/to/your/openstack-rc` to read your OpenStack credentials like `OS_AUTH_URL`, `OS_USERNAME`, `OS_PASSWORD`, etc. Those variables are used for accessing OpenStack from the external cloud provider.
 - Run the `cluster.yml` playbook

inventory/sample/group_vars/all/openstack.yml

@@ -33,6 +33,7 @@
 #   - ""
 # external_openstack_network_public_networks:
 #   - ""
+# external_openstack_metadata_search_order: "configDrive,metadataService"
 
 ## The tag of the external OpenStack Cloud Controller image
 # external_openstack_cloud_controller_image_tag: "latest"

roles/etcd/handlers/backup.yml

@@ -46,10 +46,10 @@
 - name: Backup etcd v3 data
   command: >-
     {{ bin_dir }}/etcdctl
-      --endpoints={{ etcd_access_addresses }}
       snapshot save {{ etcd_backup_directory }}/snapshot.db
   environment:
     ETCDCTL_API: 3
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
     ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
     ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"

roles/etcd/tasks/configure.yml

@@ -1,6 +1,6 @@
 ---
 - name: Configure | Check if etcd cluster is healthy
-  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
+  shell: "{{ bin_dir }}/etcdctl cluster-health | grep -q 'cluster is healthy'"
   register: etcd_cluster_is_healthy
   failed_when: false
   changed_when: false
@@ -10,12 +10,14 @@
   tags:
     - facts
   environment:
+    ETCDCTL_API: 2
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
     ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
 
 - name: Configure | Check if etcd-events cluster is healthy
-  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
+  shell: "{{ bin_dir }}/etcdctl cluster-health | grep -q 'cluster is healthy'"
   register: etcd_events_cluster_is_healthy
   failed_when: false
   changed_when: false
@@ -25,6 +27,8 @@
   tags:
     - facts
   environment:
+    ETCDCTL_API: 2
+    ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
     ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
@@ -70,7 +74,7 @@
   when: is_etcd_master and etcd_events_cluster_setup
 
 - name: Configure | Wait for etcd cluster to be healthy
-  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
+  shell: "{{ bin_dir }}/etcdctl --no-sync cluster-health | grep -q 'cluster is healthy'"
   register: etcd_cluster_is_healthy
   until: etcd_cluster_is_healthy.rc == 0
   retries: "{{ etcd_retries }}"
@@ -85,12 +89,14 @@
   tags:
     - facts
   environment:
+    ETCDCTL_API: 2
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
     ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
 
 - name: Configure | Wait for etcd-events cluster to be healthy
-  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
+  shell: "{{ bin_dir }}/etcdctl --no-sync cluster-health | grep -q 'cluster is healthy'"
   register: etcd_events_cluster_is_healthy
   until: etcd_events_cluster_is_healthy.rc == 0
   retries: "{{ etcd_retries }}"
@@ -105,12 +111,14 @@
   tags:
     - facts
   environment:
+    ETCDCTL_API: 2
+    ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
     ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
 
 - name: Configure | Check if member is in etcd cluster
-  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
+  shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}"
   register: etcd_member_in_cluster
   ignore_errors: true
   changed_when: false
@@ -119,12 +127,14 @@
   tags:
     - facts
   environment:
+    ETCDCTL_API: 2
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
     ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
 
 - name: Configure | Check if member is in etcd-events cluster
-  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_access_address }}"
+  shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}"
   register: etcd_events_member_in_cluster
   ignore_errors: true
   changed_when: false
@@ -133,6 +143,8 @@
   tags:
     - facts
   environment:
+    ETCDCTL_API: 2
+    ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
     ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"

roles/etcd/tasks/join_etcd-events_member.yml

@@ -1,11 +1,13 @@
 ---
 - name: Join Member | Add member to etcd-events cluster
-  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} member add {{ etcd_member_name }} {{ etcd_events_peer_url }}"
+  shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} {{ etcd_events_peer_url }}"
   register: member_add_result
   until: member_add_result.rc == 0
   retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   environment:
+    ETCDCTL_API: 2
+    ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
 
@@ -22,13 +24,15 @@
       {%- endfor -%}
 
 - name: Join Member | Ensure member is in etcd-events cluster
-  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_events_access_address }}"
+  shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_events_access_address }}"
   register: etcd_events_member_in_cluster
   changed_when: false
   check_mode: no
   tags:
     - facts
   environment:
+    ETCDCTL_API: 2
+    ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
 

roles/etcd/tasks/join_etcd_member.yml

@@ -1,11 +1,13 @@
 ---
 - name: Join Member | Add member to etcd cluster
-  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}"
+  shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} {{ etcd_peer_url }}"
   register: member_add_result
   until: member_add_result.rc == 0
   retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   environment:
+    ETCDCTL_API: 2
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
     ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
@@ -23,13 +25,15 @@
       {%- endfor -%}
 
 - name: Join Member | Ensure member is in etcd cluster
-  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
+  shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}"
   register: etcd_member_in_cluster
   changed_when: false
   check_mode: no
   tags:
     - facts
   environment:
+    ETCDCTL_API: 2
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
     ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"

roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config.j2

@@ -57,3 +57,8 @@ internal-network-name="{{ network_name }}"
 {% for network_name in external_openstack_network_public_networks %}
 public-network-name="{{ network_name }}"
 {% endfor %}
+
+[Metadata]
+{% if external_openstack_metadata_search_order is defined %}
+search-order="{{ external_openstack_metadata_search_order }}"
+{% endif %}

roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2

@@ -57,6 +57,7 @@ rules:
       - blockaffinities
       - ipamblocks
       - ipamhandles
+      - hostendpoints
     verbs:
       - get
       - list
@@ -72,3 +73,18 @@ rules:
       - create
       - update
 {% endif %}
+{% if calico_version is version('v3.14.0', '>=') %}
+  # KubeControllersConfiguration is where it gets its config
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - kubecontrollersconfigurations
+    verbs:
+      # read its own config
+      - get
+      # create a default if none exists
+      - create
+      # update status
+      - update
+      # watch for changes
+      - watch
+{% endif %}

roles/network_plugin/calico/templates/kdd-crds.yml.j2

@@ -2740,3 +2740,18 @@ spec:
     served: true
     storage: true
 {% endif %}
+{% if calico_version is version('v3.14.0', '>=') %}
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: kubecontrollersconfigurations.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: KubeControllersConfiguration
+    plural: kubecontrollersconfigurations
+    singular: kubecontrollersconfiguration
+{% endif %}

roles/recover_control_plane/etcd/tasks/main.yml

@@ -1,12 +1,16 @@
 ---
 - name: Get etcd endpoint health
-  shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem --endpoints={{ etcd_access_addresses }} endpoint health"
+  shell: "{{ bin_dir }}/etcdctl endpoint health"
   register: etcd_endpoint_health
   ignore_errors: true
   changed_when: false
   check_mode: no
   environment:
-    - ETCDCTL_API: 3
+    ETCDCTL_API: 3
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
   when:
     - groups['broken_etcd']
 
@@ -53,21 +57,29 @@
     - "item.rc != 0 and not 'No such file or directory' in item.stderr"
 
 - name: Get etcd cluster members
-  shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem member list"
+  shell: "{{ bin_dir }}/etcdctl member list"
   register: member_list
   changed_when: false
   check_mode: no
   environment:
-    - ETCDCTL_API: 3
+    ETCDCTL_API: 3
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
   when:
     - groups['broken_etcd']
     - not healthy
     - has_quorum
 
 - name: Remove broken cluster members
-  shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem --endpoints={{ etcd_access_addresses }} member remove {{ item[1].replace(' ','').split(',')[0] }}"
+  shell: "{{ bin_dir }}/etcdctl member remove {{ item[1].replace(' ','').split(',')[0] }}"
   environment:
-    - ETCDCTL_API: 3
+    ETCDCTL_API: 3
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
   with_nested:
     - "{{ groups['broken_etcd'] }}"
     - "{{ member_list.stdout_lines }}"

roles/remove-node/remove-etcd-node/tasks/main.yml

@@ -6,14 +6,16 @@
     - inventory_hostname in groups['etcd']
 
 - name: Lookup etcd member id
-  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep {{ node_ip }} | cut -d: -f1"
+  shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep {{ node_ip }} | cut -d: -f1"
   register: etcd_member_id
   ignore_errors: true
   changed_when: false
   check_mode: no
   tags:
     - facts
   environment:
+    ETCDCTL_API: 2
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}-key.pem"
     ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
@@ -22,7 +24,7 @@
     - inventory_hostname in groups['etcd']
 
 - name: Remove etcd member from cluster
-  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member remove {{ etcd_member_id.stdout }}"
+  shell: "{{ bin_dir }}/etcdctl --no-sync member remove {{ etcd_member_id.stdout }}"
   register: etcd_member_in_cluster
   ignore_errors: false
   retries: 6
@@ -33,6 +35,8 @@
   tags:
     - facts
   environment:
+    ETCDCTL_API: 2
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}-key.pem"
     ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"