add hypershift upgrade from amd64 to multi arch (#53675)

ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16__amd64-nightly.yaml

@@ -545,6 +545,19 @@ tests:
     test:
     - chain: openshift-e2e-test-hypershift-qe
     workflow: cucushift-installer-rehearse-aws-ipi-ovn-hypershift-guest
+- as: aws-ipi-ovn-hypershift-to-multiarch-f7
+  cron: 1 21 6,15,22,29 * *
+  steps:
+    cluster_profile: aws-qe
+    dependencies:
+      NODEPOOL_RELEASE_IMAGE_LATEST: release:target
+    env:
+      BASE_DOMAIN: qe.devcluster.openshift.com
+      HYPERSHIFT_ENABLE_MULTIARCH: "true"
+      TEST_FILTERS: ~DisconnectedOnly&;~HyperShiftMGMT&;~MicroShiftOnly&;~NonHyperShiftHOST&
+    test:
+    - chain: openshift-upgrade-qe-test-hypershift-to-multi
+    workflow: cucushift-installer-rehearse-aws-ipi-ovn-hypershift-guest
 - as: aws-ipi-ovn-hypershift-fips-guest-f28-destructive
   cron: 10 8 14 * *
   steps:

ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16-periodics.yaml

@@ -15544,6 +15544,105 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build05
+  cron: 1 21 6,15,22,29 * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.16
+    org: openshift
+    repo: openshift-tests-private
+  labels:
+    ci-operator.openshift.io/cloud: aws
+    ci-operator.openshift.io/cloud-cluster-profile: aws-qe
+    ci-operator.openshift.io/variant: amd64-nightly
+    ci.openshift.io/generator: prowgen
+    job-release: "4.16"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-openshift-tests-private-release-4.16-amd64-nightly-aws-ipi-ovn-hypershift-to-multiarch-f7
+  reporter_config:
+    slack:
+      channel: '#forum-prow-hypershift-qe-ci'
+      job_states_to_report:
+      - failure
+      - error
+      - success
+      report_template: '{{if eq .Status.State "success"}} :rainbow: Job *{{.Spec.Job}}*
+        ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> :rainbow: {{else}}
+        :volcano: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View
+        logs> :volcano: {{end}}'
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --oauth-token-path=/usr/local/github-credentials/oauth
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --secret-dir=/usr/local/aws-ipi-ovn-hypershift-to-multiarch-f7-cluster-profile
+      - --target=aws-ipi-ovn-hypershift-to-multiarch-f7
+      - --variant=amd64-nightly
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /usr/local/aws-ipi-ovn-hypershift-to-multiarch-f7-cluster-profile
+        name: cluster-profile
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /usr/local/github-credentials
+        name: github-credentials-openshift-ci-robot-private-git-cloner
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: cluster-profile
+      secret:
+        secretName: cluster-secrets-aws-qe
+    - name: github-credentials-openshift-ci-robot-private-git-cloner
+      secret:
+        secretName: github-credentials-openshift-ci-robot-private-git-cloner
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build05
   cron: 26 1 1,3,5,7,9,11,13,15,17,19,21,23,25,27,29 * *

ci-operator/step-registry/cucushift/upgrade/hypershift/control-plane/toimage/cucushift-upgrade-hypershift-control-plane-toimage-commands.sh

@@ -103,9 +103,17 @@ TARGET_MAIN_VERSION="$(echo "$TARGET_VERSION" | cut -d '.' -f 1-2)"
 SOURCE_MAIN_VERSION="$(oc get clusterversion --no-headers | awk '{print $2}' | cut -d '.' -f 1-2)"
 echo "TARGET_MAIN_VERSION: $TARGET_MAIN_VERSION , SOURCE_MAIN_VERSION:$SOURCE_MAIN_VERSION"
 oc annotate hostedcluster -n "$HYPERSHIFT_NAMESPACE" "$cluster_name" "hypershift.openshift.io/force-upgrade-to=${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}" --overwrite
-set +x
 oc patch hostedcluster "$cluster_name" -n "$HYPERSHIFT_NAMESPACE" --type=merge -p '{"spec":{"release":{"image":"'"${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}"'"}}}'
 
+if [[ "${HYPERSHIFT_ENABLE_MULTIARCH}" == "true" ]]; then
+  echo "target hc multiArch is true"
+  platform=$(oc get hostedcluster "$cluster_name" -n "$HYPERSHIFT_NAMESPACE" --ignore-not-found -o=jsonpath='{.spec.platform.type}')
+  if [[ "${platform}" == "AWS" ]] ; then
+    oc patch hostedcluster "$cluster_name" -n "$HYPERSHIFT_NAMESPACE" --type=merge -p '{"spec":{"platform":{"aws":{"multiArch": true}}}}'
+  fi
+fi
+set +x
+
 _upgradeReady=1
 for ((i=1; i<=120; i++)); do
     count=$(oc get hostedcluster -n "$HYPERSHIFT_NAMESPACE" "$cluster_name" -ojsonpath='{.status.version.history[?(@.image=="'"${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}"'")].state}' | grep -c Completed || true)
@@ -118,6 +126,9 @@ for ((i=1; i<=120; i++)); do
     sleep 30
 done
 
+# dump hc
+oc get hostedcluster "$cluster_name" -n "$HYPERSHIFT_NAMESPACE" -oyaml > "${ARTIFACT_DIR}/hostedcluster.yaml"
+
 if [ $_upgradeReady -ne 0 ]; then
     echo "HyperShift HostedCluster(CP) upgrade failed"
     exit 1

ci-operator/step-registry/cucushift/upgrade/hypershift/control-plane/toimage/cucushift-upgrade-hypershift-control-plane-toimage-ref.yaml

@@ -14,6 +14,9 @@ ref:
   - name: HYPERSHIFT_NAMESPACE
     default: "clusters"
     documentation: "The Namespace where to create the HostedCluster and NodePools"
+  - name: HYPERSHIFT_ENABLE_MULTIARCH
+    default: "false"
+    documentation: "hypershift hostedcluster upgrade target architecture, true or false"
   dependencies:
   - name: "release:target"
     env: "OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE"

ci-operator/step-registry/hypershift/aws/create/nodepool/hypershift-aws-create-nodepool-commands.sh

@@ -2,15 +2,22 @@
 
 set -exuo pipefail
 
+if [ ! -f "${SHARED_DIR}/mgmt_kubeconfig" ]; then
+    exit 1
+fi
+echo "switch kubeconfig"
+export KUBECONFIG="${SHARED_DIR}/mgmt_kubeconfig"
+
 CLUSTER_NAME="$(echo -n $PROW_JOB_ID|sha256sum|cut -c-20)"
 echo "$(date) Creating additional NodePool for HyperShift cluster ${CLUSTER_NAME}"
+
 /usr/bin/hypershift create nodepool aws \
   --cluster-name  ${CLUSTER_NAME} \
   --name additional-${CLUSTER_NAME} \
   --node-count ${ADDITIONAL_HYPERSHIFT_NODE_COUNT} \
   --instance-type ${ADDITIONAL_HYPERSHIFT_INSTANCE_TYPE} \
   --arch ${ADDITIONAL_HYPERSHIFT_NODE_ARCH} \
-  --release-image ${RELEASE_IMAGE_LATEST}
+  --release-image ${NODEPOOL_RELEASE_IMAGE_LATEST}
 
 echo "Wait additional nodepool ready..."
 oc wait --timeout=30m nodepool -n clusters additional-${CLUSTER_NAME} --for=condition=Ready
@@ -21,4 +28,6 @@ until \
     echo "$(date --rfc-3339=seconds) Clusteroperators not yet ready"
     oc get clusterversion 2>/dev/null || true
     sleep 10s
-done
+done
+
+export KUBECONFIG="${SHARED_DIR}/kubeconfig"

ci-operator/step-registry/hypershift/aws/create/nodepool/hypershift-aws-create-nodepool-ref.yaml

@@ -21,7 +21,7 @@ ref:
   timeout: 35m0s
   dependencies:
   - name: "release:latest"
-    env: RELEASE_IMAGE_LATEST
+    env: NODEPOOL_RELEASE_IMAGE_LATEST
   - name: hypershift-operator
     env: HYPERSHIFT_RELEASE_LATEST
 

ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/OWNERS

@@ -0,0 +1,9 @@
+approvers:
+- jhou1
+- liangxia
+- jianzhangbjz
+- kuiwang02
+- jiajliu
+- shellyyang1989
+- aleskandro
+- LiangquanLi930

ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/openshift-upgrade-qe-test-hypershift-to-multi-chain.metadata.json

@@ -0,0 +1,15 @@
+{
+	"path": "openshift/upgrade/qe/test/hypershift/to-multi/openshift-upgrade-qe-test-hypershift-to-multi-chain.yaml",
+	"owners": {
+		"approvers": [
+			"jhou1",
+			"liangxia",
+			"jianzhangbjz",
+			"kuiwang02",
+			"jiajliu",
+			"shellyyang1989",
+			"aleskandro",
+			"LiangquanLi930"
+		]
+	}
+}

ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/openshift-upgrade-qe-test-hypershift-to-multi-chain.yaml

@@ -0,0 +1,16 @@
+chain:
+  as: openshift-upgrade-qe-test-hypershift-to-multi
+  steps:
+  - ref: cucushift-pre
+  - ref: openshift-extended-upgrade-pre
+  - ref: cucushift-upgrade-prepare
+  - ref: cucushift-upgrade-hypershift-control-plane-toimage
+  - ref: hypershift-aws-create-nodepool
+  - ref: openshift-extended-upgrade-post
+  - ref: cucushift-upgrade-check
+  - ref: openshift-upgrade-qe-test-report
+  documentation: |-
+    The chain executes upgrade-prepare, which creates test data on the cluster;
+    then upgrade the HyperShift HostedCluster(CP) to multi-arch release;
+    then create an arm64 nodepool;
+    then execute upgrade-check, which checks the prepared test data.