Use cloud-provider template for PowerVS e2e by default (#1861)

Amulyam24 · web-flow · commit a34ad8cd621f · 2024-07-01T22:06:02.000-07:00
diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh
@@ -128,6 +128,12 @@ prerequisites_powervs(){
     export IBMPOWERVS_SERVICE_INSTANCE_ID=${BOSKOS_RESOURCE_ID:-"d53da3bf-1f4a-42fa-9735-acf16b1a05cd"}
     export IBMPOWERVS_NETWORK_NAME="capi-net-$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | head --bytes 5)"
     export ZONE=${BOSKOS_ZONE:-"osa21"}
+    export IBMPOWERVS_REGION=${BOSKOS_REGION:-"osa"}
+    export IBMPOWERVS_ZONE=${BOSKOS_ZONE:-"osa21"}
+    export PROVIDER_ID_FORMAT=v2
+    export EXP_CLUSTER_RESOURCE_SET=true
+    export IBMACCOUNT_ID=${IBMACCOUNT_ID:-"7cfbd5381a434af7a09289e795840d4e"}
+    export BASE64_API_KEY=$(tr -d '\n' <<<"$IBMCLOUD_API_KEY" | base64)
 }
 
 prerequisites_vpc(){
diff --git a/test/e2e/data/templates/cluster-template-powervs-md-remediation.yaml b/test/e2e/data/templates/cluster-template-powervs-md-remediation.yaml
@@ -1,8 +1,273 @@
+apiVersion: v1
+data:
+  ibmpowervs-ccm-external.yaml: |-
+    apiVersion: v1
+    kind: ServiceAccount
+    metadata:
+      name: cloud-controller-manager
+      namespace: kube-system
+    ---
+    apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata:
+      name: cloud-controller-manager:apiserver-authentication-reader
+      namespace: kube-system
+    roleRef:
+      apiGroup: rbac.authorization.k8s.io
+      kind: Role
+      name: extension-apiserver-authentication-reader
+    subjects:
+      - apiGroup: ""
+        kind: ServiceAccount
+        name: cloud-controller-manager
+        namespace: kube-system
+    ---
+    apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRoleBinding
+    metadata:
+      name: system:cloud-controller-manager
+    roleRef:
+      apiGroup: rbac.authorization.k8s.io
+      kind: ClusterRole
+      name: system:cloud-controller-manager
+    subjects:
+      - kind: ServiceAccount
+        name: cloud-controller-manager
+        namespace: kube-system
+    ---
+    apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+      name: system:cloud-controller-manager
+    rules:
+      - apiGroups:
+          - ""
+        resources:
+          - events
+        verbs:
+          - create
+          - patch
+          - update
+      - apiGroups:
+          - ""
+        resources:
+          - nodes
+        verbs:
+          - "*"
+      - apiGroups:
+          - ""
+        resources:
+          - nodes/status
+        verbs:
+          - patch
+      - apiGroups:
+          - ""
+        resources:
+          - services
+        verbs:
+          - list
+          - patch
+          - update
+          - watch
+      - apiGroups:
+          - ""
+        resources:
+          - services/status
+        verbs:
+          - patch
+      - apiGroups:
+          - ""
+        resources:
+          - serviceaccounts
+        verbs:
+          - create
+          - get
+          - list
+          - watch
+          - update
+      - apiGroups:
+          - ""
+        resources:
+          - persistentvolumes
+        verbs:
+          - get
+          - list
+          - update
+          - watch
+      - apiGroups:
+          - ""
+        resources:
+          - endpoints
+        verbs:
+          - create
+          - get
+          - list
+          - watch
+          - update
+      - apiGroups:
+          - ""
+        resources:
+          - secrets
+        verbs:
+          - get
+          - list
+          - watch
+      - apiGroups:
+          - "coordination.k8s.io"
+        resources:
+          - leases
+        verbs:
+          - create
+          - get
+          - list
+          - watch
+          - update
+      - apiGroups:
+          - ""
+        resourceNames:
+          - node-controller
+          - service-controller
+        resources:
+          - serviceaccounts/token
+        verbs:
+          - create
+    ---
+    apiVersion: apps/v1
+    kind: DaemonSet
+    metadata:
+      name: ibmpowervs-cloud-controller-manager
+      namespace: kube-system
+      labels:
+        k8s-app: ibmpowervs-cloud-controller-manager
+    spec:
+      selector:
+        matchLabels:
+          k8s-app: ibmpowervs-cloud-controller-manager
+      updateStrategy:
+        type: RollingUpdate
+      template:
+        metadata:
+          labels:
+            k8s-app: ibmpowervs-cloud-controller-manager
+        spec:
+          nodeSelector:
+            node-role.kubernetes.io/control-plane: ""
+          tolerations:
+            - key: node.cloudprovider.kubernetes.io/uninitialized
+              value: "true"
+              effect: NoSchedule
+            - key: node-role.kubernetes.io/master
+              effect: NoSchedule
+              operator: Exists
+            - key: node-role.kubernetes.io/control-plane
+              effect: NoSchedule
+              operator: Exists
+            - key: node.kubernetes.io/not-ready
+              effect: NoSchedule
+              operator: Exists
+          serviceAccountName: cloud-controller-manager
+          containers:
+            - name: ibmpowervs-cloud-controller-manager
+              image: gcr.io/k8s-staging-capi-ibmcloud/powervs-cloud-controller-manager:6c98ec5
+              args:
+                - --v=2
+                - --cloud-provider=ibm
+                - --cloud-config=/etc/cloud/ibmpowervs.conf
+                - --use-service-account-credentials=true
+              env:
+                - name: ENABLE_VPC_PUBLIC_ENDPOINT
+                  value: "true"
+              resources:
+                requests:
+                  cpu: 200m
+              terminationMessagePolicy: FallbackToLogsOnError
+              volumeMounts:
+                - mountPath: /etc/cloud
+                  name: ibmpowervs-config-volume
+                  readOnly: true
+                - mountPath: /etc/ibm-secret
+                  name: ibm-secret
+          hostNetwork: true
+          volumes:
+            - name: ibmpowervs-config-volume
+              configMap:
+                name: ibmpowervs-cloud-config
+            - name: ibm-secret
+              secret:
+                secretName: ibmpowervs-cloud-credential
+kind: ConfigMap
+metadata:
+  name: cloud-controller-manager-addon
+---
+apiVersion: v1
+data:
+  ibmpowervs-cloud-conf.yaml: |-
+    apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: ibmpowervs-cloud-config
+      namespace: kube-system
+    data:
+      ibmpowervs.conf: |
+        [global]
+        version = 1.1.0
+        [kubernetes]
+        config-file = ""
+        [provider]
+        cluster-default-provider = g2
+        accountID = ${IBMACCOUNT_ID}
+        clusterID = ${CLUSTER_NAME}
+        g2workerServiceAccountID = ${IBMACCOUNT_ID}
+        g2Credentials = /etc/ibm-secret/ibmcloud_api_key
+        g2ResourceGroupName = ${IBMVPC_RESOURCE_GROUP:=""}
+        g2VpcSubnetNames = ${IBMVPC_SUBNET_NAMES:=""}
+        g2VpcName = ${IBMVPC_NAME:=""}
+        region =  ${IBMVPC_REGION:=""}
+        powerVSCloudInstanceID = ${IBMPOWERVS_SERVICE_INSTANCE_ID}
+        powerVSRegion = ${IBMPOWERVS_REGION}
+        powerVSZone = ${IBMPOWERVS_ZONE}
+kind: ConfigMap
+metadata:
+  name: ibmpowervs-cfg
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ibmpowervs-credential
+stringData:
+  ibmpowervs-credential.yaml: |-
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: ibmpowervs-cloud-credential
+      namespace: kube-system
+    data:
+      ibmcloud_api_key: ${BASE64_API_KEY}
+type: addons.cluster.x-k8s.io/resource-set
+---
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+  name: crs-cloud-conf
+spec:
+  clusterSelector:
+    matchLabels:
+      ccm: external
+  resources:
+  - kind: Secret
+    name: ibmpowervs-credential
+  - kind: ConfigMap
+    name: ibmpowervs-cfg
+  - kind: ConfigMap
+    name: cloud-controller-manager-addon
+  strategy: ApplyOnce
+---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: KubeadmConfigTemplate
 metadata:
   labels:
     cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+    cluster.x-k8s.io/control-plane: ""
   name: ${CLUSTER_NAME}-md-0
 spec:
   template:
@@ -19,7 +284,6 @@ spec:
           kubeletExtraArgs:
             cloud-provider: external
             eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
-            provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}'
           name: '{{ v1.local_hostname }}'
       preKubeadmCommands:
       - hostname "{{ v1.local_hostname }}"
@@ -32,6 +296,7 @@ apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:
   labels:
+    ccm: external
     cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
   name: ${CLUSTER_NAME}
 spec:
@@ -77,25 +342,6 @@ spec:
         name: ${CLUSTER_NAME}-md-0
       version: ${KUBERNETES_VERSION}
 ---
-apiVersion: cluster.x-k8s.io/v1beta1
-kind: MachineHealthCheck
-metadata:
-  name: ${CLUSTER_NAME}-mhc-md
-spec:
-  clusterName: ${CLUSTER_NAME}
-  maxUnhealthy: 100%
-  nodeStartupTimeout: 20m
-  selector:
-    matchLabels:
-      e2e.remediation.label: ""
-  unhealthyConditions:
-  - status: "False"
-    timeout: 60s
-    type: Ready
-  - status: Unknown
-    timeout: 60s
-    type: Ready
----
 apiVersion: controlplane.cluster.x-k8s.io/v1beta1
 kind: KubeadmControlPlane
 metadata:
@@ -107,9 +353,12 @@ spec:
         certSANs:
         - ${IBMPOWERVS_VIP}
         - ${IBMPOWERVS_VIP_EXTERNAL}
+        extraArgs:
+          cloud-provider: external
       controlPlaneEndpoint: ${IBMPOWERVS_VIP}:${API_SERVER_PORT:=6443}
       controllerManager:
         extraArgs:
+          cloud-provider: external
           enable-hostpath-provisioner: "true"
     files:
     - content: |
@@ -234,7 +483,6 @@ spec:
         kubeletExtraArgs:
           cloud-provider: external
           eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
-          provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}'
         name: '{{ v1.local_hostname }}'
     joinConfiguration:
       discovery:
@@ -248,7 +496,6 @@ spec:
         kubeletExtraArgs:
           cloud-provider: external
           eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
-          provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}'
         name: '{{ v1.local_hostname }}'
     preKubeadmCommands:
     - hostname "{{ v1.local_hostname }}"
diff --git a/test/e2e/data/templates/cluster-template-powervs-md-remediation/kustomization.yaml b/test/e2e/data/templates/cluster-template-powervs-md-remediation/kustomization.yaml
@@ -1,7 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ../../../../../templates/cluster-template-powervs.yaml
-  - mhc-md-powervs.yaml
+  - ../../../../../templates/cluster-template-powervs-cloud-provider.yaml
 patchesStrategicMerge:
   - patches/mhc-label.yaml
