From d866689d38b45a6ec88f9648ec95486f7be48d72 Mon Sep 17 00:00:00 2001 From: He Liu Date: Wed, 26 Jun 2024 18:10:11 +0800 Subject: [PATCH] add hypershift upgrade from amd64 to multi arch --- ...s-private-release-4.16__amd64-nightly.yaml | 12 +++ ...-tests-private-release-4.16-periodics.yaml | 88 +++++++++++++++++++ ...pershift-control-plane-toimage-commands.sh | 8 ++ ...-hypershift-control-plane-toimage-ref.yaml | 3 + ...hypershift-aws-create-nodepool-commands.sh | 14 ++- .../hypershift-aws-create-nodepool-ref.yaml | 3 + .../qe/test/hypershift/to-multi/OWNERS | 9 ++ ...st-hypershift-to-multi-chain.metadata.json | 15 ++++ ...ade-qe-test-hypershift-to-multi-chain.yaml | 16 ++++ 9 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/OWNERS create mode 100644 ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/openshift-upgrade-qe-test-hypershift-to-multi-chain.metadata.json create mode 100644 ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/openshift-upgrade-qe-test-hypershift-to-multi-chain.yaml diff --git a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16__amd64-nightly.yaml b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16__amd64-nightly.yaml index c774188230712..5efc4236ceaec 100644 --- a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16__amd64-nightly.yaml +++ b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16__amd64-nightly.yaml @@ -545,6 +545,18 @@ tests: test: - chain: openshift-e2e-test-hypershift-qe workflow: cucushift-installer-rehearse-aws-ipi-ovn-hypershift-guest +- as: aws-ipi-ovn-hypershift-to-multiarch-f7 + cron: 1 21 6,15,22,29 * * + steps: + cluster_profile: aws-qe + env: + ADDITIONAL_HYPERSHIFT_NODE_RELEASE: quay.io/openshift-release-dev/ocp-release:4.16.1-multi + BASE_DOMAIN: qe.devcluster.openshift.com + HYPERSHIFT_TARGET_MULTIARCH: "true" + TEST_FILTERS: ~DisconnectedOnly&;~HyperShiftMGMT&;~MicroShiftOnly&;~NonHyperShiftHOST& + test: + - chain: openshift-upgrade-qe-test-hypershift-to-multi + workflow: cucushift-installer-rehearse-aws-ipi-ovn-hypershift-guest - as: aws-ipi-ovn-hypershift-fips-guest-f28-destructive cron: 10 8 14 * * steps: diff --git a/ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16-periodics.yaml b/ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16-periodics.yaml index fb6f7b9c311ad..3c3798ffb409f 100644 --- a/ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16-periodics.yaml +++ b/ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16-periodics.yaml @@ -15544,6 +15544,94 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build05 + cron: 1 21 6,15,22,29 * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: release-4.16 + org: openshift + repo: openshift-tests-private + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: aws-qe + ci-operator.openshift.io/variant: amd64-nightly + ci.openshift.io/generator: prowgen + job-release: "4.16" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-openshift-tests-private-release-4.16-amd64-nightly-aws-ipi-ovn-hypershift-to-multiarch-f7 + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --oauth-token-path=/usr/local/github-credentials/oauth + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --secret-dir=/usr/local/aws-ipi-ovn-hypershift-to-multiarch-f7-cluster-profile + - --target=aws-ipi-ovn-hypershift-to-multiarch-f7 + - --variant=amd64-nightly + command: + - ci-operator + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /usr/local/aws-ipi-ovn-hypershift-to-multiarch-f7-cluster-profile + name: cluster-profile + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: cluster-profile + secret: + secretName: cluster-secrets-aws-qe + - name: github-credentials-openshift-ci-robot-private-git-cloner + secret: + secretName: github-credentials-openshift-ci-robot-private-git-cloner + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build05 cron: 26 1 1,3,5,7,9,11,13,15,17,19,21,23,25,27,29 * * diff --git a/ci-operator/step-registry/cucushift/upgrade/hypershift/control-plane/toimage/cucushift-upgrade-hypershift-control-plane-toimage-commands.sh b/ci-operator/step-registry/cucushift/upgrade/hypershift/control-plane/toimage/cucushift-upgrade-hypershift-control-plane-toimage-commands.sh index 12910477cfad5..9cb39afd4fa35 100644 --- a/ci-operator/step-registry/cucushift/upgrade/hypershift/control-plane/toimage/cucushift-upgrade-hypershift-control-plane-toimage-commands.sh +++ b/ci-operator/step-registry/cucushift/upgrade/hypershift/control-plane/toimage/cucushift-upgrade-hypershift-control-plane-toimage-commands.sh @@ -106,6 +106,14 @@ oc annotate hostedcluster -n "$HYPERSHIFT_NAMESPACE" "$cluster_name" "hypershift set +x oc patch hostedcluster "$cluster_name" -n "$HYPERSHIFT_NAMESPACE" --type=merge -p '{"spec":{"release":{"image":"'"${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}"'"}}}' +if [ -n "${HYPERSHIFT_TARGET_MULTIARCH}" ]; then + echo "target hc multiarch is ${HYPERSHIFT_TARGET_MULTIARCH}" + platform=$(oc get hostedcluster "$cluster_name" -n "$HYPERSHIFT_NAMESPACE" --ignore-not-found -o=jsonpath='{.spec.platform.type}') + if [ "${platform}" == "AWS" ] ; then + oc patch hostedcluster "$cluster_name" -n "$HYPERSHIFT_NAMESPACE" --type=merge -p '{"spec":{"platform":{"aws":{"multiArch": "'"${HYPERSHIFT_TARGET_MULTIARCH}"'"}}}}' + fi +fi + _upgradeReady=1 for ((i=1; i<=120; i++)); do count=$(oc get hostedcluster -n "$HYPERSHIFT_NAMESPACE" "$cluster_name" -ojsonpath='{.status.version.history[?(@.image=="'"${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}"'")].state}' | grep -c Completed || true) diff --git a/ci-operator/step-registry/cucushift/upgrade/hypershift/control-plane/toimage/cucushift-upgrade-hypershift-control-plane-toimage-ref.yaml b/ci-operator/step-registry/cucushift/upgrade/hypershift/control-plane/toimage/cucushift-upgrade-hypershift-control-plane-toimage-ref.yaml index 0e519bc36a0d2..44f40bc65c665 100644 --- a/ci-operator/step-registry/cucushift/upgrade/hypershift/control-plane/toimage/cucushift-upgrade-hypershift-control-plane-toimage-ref.yaml +++ b/ci-operator/step-registry/cucushift/upgrade/hypershift/control-plane/toimage/cucushift-upgrade-hypershift-control-plane-toimage-ref.yaml @@ -14,6 +14,9 @@ ref: - name: HYPERSHIFT_NAMESPACE default: "clusters" documentation: "The Namespace where to create the HostedCluster and NodePools" + - name: HYPERSHIFT_TARGET_MULTIARCH + default: "" + documentation: "hypershift hostedcluster upgrade target architecture, true or false" dependencies: - name: "release:target" env: "OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE" diff --git a/ci-operator/step-registry/hypershift/aws/create/nodepool/hypershift-aws-create-nodepool-commands.sh b/ci-operator/step-registry/hypershift/aws/create/nodepool/hypershift-aws-create-nodepool-commands.sh index 13edb32105118..9b50a533a39f4 100755 --- a/ci-operator/step-registry/hypershift/aws/create/nodepool/hypershift-aws-create-nodepool-commands.sh +++ b/ci-operator/step-registry/hypershift/aws/create/nodepool/hypershift-aws-create-nodepool-commands.sh @@ -2,15 +2,23 @@ set -exuo pipefail +if [ ! -f "${SHARED_DIR}/mgmt_kubeconfig" ]; then + exit 1 +fi +echo "switch kubeconfig" +export KUBECONFIG="${SHARED_DIR}/mgmt_kubeconfig" + CLUSTER_NAME="$(echo -n $PROW_JOB_ID|sha256sum|cut -c-20)" echo "$(date) Creating additional NodePool for HyperShift cluster ${CLUSTER_NAME}" + +RELEASE_IMAGE=${ADDITIONAL_HYPERSHIFT_NODE_RELEASE:-$RELEASE_IMAGE_LATEST} /usr/bin/hypershift create nodepool aws \ --cluster-name ${CLUSTER_NAME} \ --name additional-${CLUSTER_NAME} \ --node-count ${ADDITIONAL_HYPERSHIFT_NODE_COUNT} \ --instance-type ${ADDITIONAL_HYPERSHIFT_INSTANCE_TYPE} \ --arch ${ADDITIONAL_HYPERSHIFT_NODE_ARCH} \ - --release-image ${RELEASE_IMAGE_LATEST} + --release-image ${RELEASE_IMAGE} echo "Wait additional nodepool ready..." oc wait --timeout=30m nodepool -n clusters additional-${CLUSTER_NAME} --for=condition=Ready @@ -21,4 +29,6 @@ until \ echo "$(date --rfc-3339=seconds) Clusteroperators not yet ready" oc get clusterversion 2>/dev/null || true sleep 10s -done \ No newline at end of file +done + +export KUBECONFIG="${SHARED_DIR}/kubeconfig" \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/aws/create/nodepool/hypershift-aws-create-nodepool-ref.yaml b/ci-operator/step-registry/hypershift/aws/create/nodepool/hypershift-aws-create-nodepool-ref.yaml index 2bbcc51086725..c62aae0e8daf6 100644 --- a/ci-operator/step-registry/hypershift/aws/create/nodepool/hypershift-aws-create-nodepool-ref.yaml +++ b/ci-operator/step-registry/hypershift/aws/create/nodepool/hypershift-aws-create-nodepool-ref.yaml @@ -12,6 +12,9 @@ ref: - name: ADDITIONAL_HYPERSHIFT_NODE_ARCH default: "arm64" documentation: "arch of nodepool, default value is for arm64." + - name: ADDITIONAL_HYPERSHIFT_NODE_RELEASE + default: "" + documentation: "release of nodepool" from: hypershift-operator grace_period: 5m0s resources: diff --git a/ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/OWNERS b/ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/OWNERS new file mode 100644 index 0000000000000..d85a24f31f5e9 --- /dev/null +++ b/ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/OWNERS @@ -0,0 +1,9 @@ +approvers: +- jhou1 +- liangxia +- jianzhangbjz +- kuiwang02 +- jiajliu +- shellyyang1989 +- aleskandro +- LiangquanLi930 diff --git a/ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/openshift-upgrade-qe-test-hypershift-to-multi-chain.metadata.json b/ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/openshift-upgrade-qe-test-hypershift-to-multi-chain.metadata.json new file mode 100644 index 0000000000000..60e094e22e520 --- /dev/null +++ b/ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/openshift-upgrade-qe-test-hypershift-to-multi-chain.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "openshift/upgrade/qe/test/hypershift/to-multi/openshift-upgrade-qe-test-hypershift-to-multi-chain.yaml", + "owners": { + "approvers": [ + "jhou1", + "liangxia", + "jianzhangbjz", + "kuiwang02", + "jiajliu", + "shellyyang1989", + "aleskandro", + "LiangquanLi930" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/openshift-upgrade-qe-test-hypershift-to-multi-chain.yaml b/ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/openshift-upgrade-qe-test-hypershift-to-multi-chain.yaml new file mode 100644 index 0000000000000..ef77844488921 --- /dev/null +++ b/ci-operator/step-registry/openshift/upgrade/qe/test/hypershift/to-multi/openshift-upgrade-qe-test-hypershift-to-multi-chain.yaml @@ -0,0 +1,16 @@ +chain: + as: openshift-upgrade-qe-test-hypershift-to-multi + steps: + - ref: cucushift-pre + - ref: openshift-extended-upgrade-pre + - ref: cucushift-upgrade-prepare + - ref: cucushift-upgrade-hypershift-control-plane-toimage + - ref: hypershift-aws-create-nodepool + - ref: openshift-extended-upgrade-post + - ref: cucushift-upgrade-check + - ref: openshift-upgrade-qe-test-report + documentation: |- + The chain executes upgrade-prepare, which creates test data on the cluster; + then upgrade the HyperShift HostedCluster(CP) to multi-arch release; + then create an arm64 nodepool; + then execute upgrade-check, which checks the prepared test data.