From e02243fcb36413d34dcef4942eb966e56eb63f8f Mon Sep 17 00:00:00 2001 From: chaoyang Date: Mon, 15 Jan 2024 09:30:06 +0800 Subject: [PATCH] Adding profile abut filestore with xpn (#46959) * Adding profile abut filestore with xpn * Change the filestore csi storageclass as non default * Using xpn.json --- ...s-private-release-4.15__multi-nightly.yaml | 12 ++++-- ...-tests-private-release-4.15-periodics.yaml | 16 ++++---- .../gcp/ipi/xpn/private/filestore-csi/OWNERS | 9 +++++ ...ivate-filestore-csi-workflow.metadata.json | 16 ++++++++ ...pi-xpn-private-filestore-csi-workflow.yaml | 18 +++++++++ ...conf-csi-optional-gcp-filestore-chain.yaml | 3 -- ...orage-create-csi-gcp-filestore-commands.sh | 37 +++++++++++++++---- .../storage-create-csi-gcp-filestore-ref.yaml | 8 +++- ...rage-destroy-csi-gcp-filestore-commands.sh | 6 +++ 9 files changed, 100 insertions(+), 25 deletions(-) create mode 100644 ci-operator/step-registry/cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/OWNERS create mode 100644 ci-operator/step-registry/cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/cucushift-installer-rehearse-gcp-ipi-xpn-private-filestore-csi-workflow.metadata.json create mode 100644 ci-operator/step-registry/cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/cucushift-installer-rehearse-gcp-ipi-xpn-private-filestore-csi-workflow.yaml diff --git a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.15__multi-nightly.yaml b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.15__multi-nightly.yaml index 4a57e7eeb4cd..e3445f6e871e 100644 --- a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.15__multi-nightly.yaml +++ b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.15__multi-nightly.yaml @@ -1965,30 +1965,34 @@ tests: test: - chain: openshift-e2e-test-qe workflow: cucushift-installer-rehearse-gcp-ipi-xpn-minimal-permission -- as: gcp-ipi-xpn-private-amd-f28-destructive +- as: gcp-ipi-xpn-private-filestore-csi-amd-f28-destructive cron: 44 6 17 * * steps: cluster_profile: gcp-qe dependencies: OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest + dependency_overrides: + OO_INDEX: quay.io/openshift-qe-optional-operators/aosqe-index:v4.15 env: E2E_RUN_TAGS: '@gcp-ipi' test: - chain: openshift-e2e-test-qe-destructive - workflow: cucushift-installer-rehearse-gcp-ipi-xpn-private -- as: gcp-ipi-xpn-private-arm-f14 + workflow: cucushift-installer-rehearse-gcp-ipi-xpn-private-filestore-csi +- as: gcp-ipi-xpn-private-filestore-csi-arm-f14 cron: 54 13 8,23 * * steps: cluster_profile: gcp-qe dependencies: OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest + dependency_overrides: + OO_INDEX: quay.io/openshift-qe-optional-operators/aosqe-index:v4.15 env: COMPUTE_NODE_TYPE: t2a-standard-4 E2E_RUN_TAGS: '@gcp-ipi' OCP_ARCH: arm64 test: - chain: openshift-e2e-test-qe - workflow: cucushift-installer-rehearse-gcp-ipi-xpn-private + workflow: cucushift-installer-rehearse-gcp-ipi-xpn-private-filestore-csi - as: gcp-ipi-xpn-oidc-amd-f28-destructive cron: 4 10 6 * * steps: diff --git a/ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.15-periodics.yaml b/ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.15-periodics.yaml index b08f9c672b80..abcdd8b76e94 100644 --- a/ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.15-periodics.yaml +++ b/ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.15-periodics.yaml @@ -52848,7 +52848,7 @@ periodics: ci.openshift.io/generator: prowgen job-release: "4.15" pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: periodic-ci-openshift-openshift-tests-private-release-4.15-multi-nightly-gcp-ipi-xpn-private-amd-f28-destructive + name: periodic-ci-openshift-openshift-tests-private-release-4.15-multi-nightly-gcp-ipi-xpn-private-filestore-csi-amd-f28-destructive spec: containers: - args: @@ -52858,8 +52858,8 @@ periodics: - --oauth-token-path=/usr/local/github-credentials/oauth - --report-credentials-file=/etc/report/credentials - --secret-dir=/secrets/ci-pull-credentials - - --secret-dir=/usr/local/gcp-ipi-xpn-private-amd-f28-destructive-cluster-profile - - --target=gcp-ipi-xpn-private-amd-f28-destructive + - --secret-dir=/usr/local/gcp-ipi-xpn-private-filestore-csi-amd-f28-destructive-cluster-profile + - --target=gcp-ipi-xpn-private-filestore-csi-amd-f28-destructive - --variant=multi-nightly command: - ci-operator @@ -52876,7 +52876,7 @@ periodics: - mountPath: /secrets/ci-pull-credentials name: ci-pull-credentials readOnly: true - - mountPath: /usr/local/gcp-ipi-xpn-private-amd-f28-destructive-cluster-profile + - mountPath: /usr/local/gcp-ipi-xpn-private-filestore-csi-amd-f28-destructive-cluster-profile name: cluster-profile - mountPath: /secrets/gcs name: gcs-credentials @@ -52936,7 +52936,7 @@ periodics: ci.openshift.io/generator: prowgen job-release: "4.15" pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: periodic-ci-openshift-openshift-tests-private-release-4.15-multi-nightly-gcp-ipi-xpn-private-arm-f14 + name: periodic-ci-openshift-openshift-tests-private-release-4.15-multi-nightly-gcp-ipi-xpn-private-filestore-csi-arm-f14 spec: containers: - args: @@ -52946,8 +52946,8 @@ periodics: - --oauth-token-path=/usr/local/github-credentials/oauth - --report-credentials-file=/etc/report/credentials - --secret-dir=/secrets/ci-pull-credentials - - --secret-dir=/usr/local/gcp-ipi-xpn-private-arm-f14-cluster-profile - - --target=gcp-ipi-xpn-private-arm-f14 + - --secret-dir=/usr/local/gcp-ipi-xpn-private-filestore-csi-arm-f14-cluster-profile + - --target=gcp-ipi-xpn-private-filestore-csi-arm-f14 - --variant=multi-nightly command: - ci-operator @@ -52964,7 +52964,7 @@ periodics: - mountPath: /secrets/ci-pull-credentials name: ci-pull-credentials readOnly: true - - mountPath: /usr/local/gcp-ipi-xpn-private-arm-f14-cluster-profile + - mountPath: /usr/local/gcp-ipi-xpn-private-filestore-csi-arm-f14-cluster-profile name: cluster-profile - mountPath: /secrets/gcs name: gcs-credentials diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/OWNERS b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/OWNERS new file mode 100644 index 000000000000..a84ea2b8087e --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/OWNERS @@ -0,0 +1,9 @@ +approvers: +- jianlinliu +- gpei +- jianli-wei +reviewers: +- jianlinliu +- gpei +- jianli-wei +- chao007 diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/cucushift-installer-rehearse-gcp-ipi-xpn-private-filestore-csi-workflow.metadata.json b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/cucushift-installer-rehearse-gcp-ipi-xpn-private-filestore-csi-workflow.metadata.json new file mode 100644 index 000000000000..d6cfa44ba06e --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/cucushift-installer-rehearse-gcp-ipi-xpn-private-filestore-csi-workflow.metadata.json @@ -0,0 +1,16 @@ +{ + "path": "cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/cucushift-installer-rehearse-gcp-ipi-xpn-private-filestore-csi-workflow.yaml", + "owners": { + "approvers": [ + "jianlinliu", + "gpei", + "jianli-wei" + ], + "reviewers": [ + "jianlinliu", + "gpei", + "jianli-wei", + "chao007" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/cucushift-installer-rehearse-gcp-ipi-xpn-private-filestore-csi-workflow.yaml b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/cucushift-installer-rehearse-gcp-ipi-xpn-private-filestore-csi-workflow.yaml new file mode 100644 index 000000000000..2d09fb52d7b3 --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/ipi/xpn/private/filestore-csi/cucushift-installer-rehearse-gcp-ipi-xpn-private-filestore-csi-workflow.yaml @@ -0,0 +1,18 @@ +workflow: + as: cucushift-installer-rehearse-gcp-ipi-xpn-private-filestore-csi + steps: + pre: + - chain: cucushift-installer-rehearse-gcp-ipi-xpn-private-provision + - chain: storage-conf-csi-optional-gcp-filestore + post: + - ref: storage-destroy-csi-gcp-filestore + - chain: cucushift-installer-rehearse-gcp-ipi-private-deprovision + documentation: |- + The IPI XPN workflow provides provision- and deprovision- steps that provision and + deprovision an OpenShift XPN private cluster with a default configuration + on GCP, allowing job authors to inject their own end-to-end test logic. + + All modifications to this workflow should be done by modifying the + `cucushift-installer-rehearse-gcp-ipi-xpn-private-provision` and `cucushift-installer-rehearse-gcp-ipi-private-deprovision` chains to + allow other workflows to mimic and extend this base workflow without + a need to backport changes. diff --git a/ci-operator/step-registry/storage/conf/csi-optional/gcp-filestore/storage-conf-csi-optional-gcp-filestore-chain.yaml b/ci-operator/step-registry/storage/conf/csi-optional/gcp-filestore/storage-conf-csi-optional-gcp-filestore-chain.yaml index cbff03da214c..7bab9c139662 100644 --- a/ci-operator/step-registry/storage/conf/csi-optional/gcp-filestore/storage-conf-csi-optional-gcp-filestore-chain.yaml +++ b/ci-operator/step-registry/storage/conf/csi-optional/gcp-filestore/storage-conf-csi-optional-gcp-filestore-chain.yaml @@ -4,10 +4,7 @@ chain: - ref: optional-operators-subscribe - ref: storage-create-csi-gcp-filestore - ref: storage-conf-wait-for-csi-driver - - ref: storage-conf-storageclass-set-default-storageclass env: - - name: REQUIRED_DEFAULT_STORAGECLASS - default: "filestore-csi" - name: CLUSTERCSIDRIVER default: filestore.csi.storage.gke.io - name: OO_PACKAGE diff --git a/ci-operator/step-registry/storage/create/csi-gcp-filestore/storage-create-csi-gcp-filestore-commands.sh b/ci-operator/step-registry/storage/create/csi-gcp-filestore/storage-create-csi-gcp-filestore-commands.sh index 5ae85465dffc..484b866b7053 100644 --- a/ci-operator/step-registry/storage/create/csi-gcp-filestore/storage-create-csi-gcp-filestore-commands.sh +++ b/ci-operator/step-registry/storage/create/csi-gcp-filestore/storage-create-csi-gcp-filestore-commands.sh @@ -3,25 +3,38 @@ set -o errexit set -o nounset set -o pipefail -CLUSTER_NAME="$(oc get -o jsonpath='{.status.infrastructureName}{"\n"}' infrastructure cluster)" -NETWORK_NAME="$CLUSTER_NAME-network" -export CLUSTER_NAME -export NETWORK_NAME -export STORAGECLASS_LOCATION=${SHARED_DIR}/filestore-sc.yaml -export MANIFEST_LOCATION=${SHARED_DIR}/${TEST_CSI_DRIVER_MANIFEST} - # For disconnected or otherwise unreachable environments, we want to # have steps use an HTTP(S) proxy to reach the API server. This proxy # configuration file should export HTTP_PROXY, HTTPS_PROXY, and NO_PROXY # environment variables, as well as their lowercase equivalents (note # that libcurl doesn't recognize the uppercase variables). -if test -f "${SHARED_DIR}/proxy-conf.sh" + +if [[ -f "${SHARED_DIR}/proxy-conf.sh" ]] then # shellcheck disable=SC1090 source "${SHARED_DIR}/proxy-conf.sh" fi +CLUSTER_NAME="$(oc get -o jsonpath='{.status.infrastructureName}{"\n"}' infrastructure cluster)" +if [[ -s "${SHARED_DIR}/xpn.json" ]] +then + echo "Reading variables from 'xpn_project_setting.json'..." + cat ${CLUSTER_PROFILE_DIR}/xpn_project_setting.json + HOST_PROJECT=$(jq -r '.hostProject' "${CLUSTER_PROFILE_DIR}/xpn_project_setting.json") + HOST_PROJECT_NETWORK=$(jq -r '.clusterNetwork' "${CLUSTER_PROFILE_DIR}/xpn_project_setting.json") + NETWORK=$(basename ${HOST_PROJECT_NETWORK}) + NETWORK_NAME=projects/${HOST_PROJECT}/global/networks/${NETWORK} +else + NETWORK_NAME="$CLUSTER_NAME-network" +fi + +export CLUSTER_NAME +export NETWORK_NAME +export STORAGECLASS_LOCATION=${SHARED_DIR}/filestore-sc.yaml +export MANIFEST_LOCATION=${SHARED_DIR}/${TEST_CSI_DRIVER_MANIFEST} + # Create StorageClass +# shared vpc, parameter should add "connect-mode: PRIVATE_SERVICE_ACCESS" echo "Creating a StorageClass" cat <>$STORAGECLASS_LOCATION apiVersion: storage.k8s.io/v1 @@ -32,10 +45,16 @@ provisioner: filestore.csi.storage.gke.io volumeBindingMode: WaitForFirstConsumer allowVolumeExpansion: true parameters: + connect-mode: DIRECT_PEERING network: $NETWORK_NAME labels: kubernetes-io-cluster-$CLUSTER_NAME=owned EOF +if [[ -s "${SHARED_DIR}/xpn.json" ]] +then + sed -i 's/DIRECT_PEERING/PRIVATE_SERVICE_ACCESS/' $STORAGECLASS_LOCATION +fi + echo "Using StorageClass file ${STORAGECLASS_LOCATION}" cat ${STORAGECLASS_LOCATION} @@ -90,3 +109,5 @@ EOF echo "Using manifest file ${MANIFEST_LOCATION}" cat ${MANIFEST_LOCATION} + +oc get sc/filestore-csi -o yaml diff --git a/ci-operator/step-registry/storage/create/csi-gcp-filestore/storage-create-csi-gcp-filestore-ref.yaml b/ci-operator/step-registry/storage/create/csi-gcp-filestore/storage-create-csi-gcp-filestore-ref.yaml index 4998b15a4d61..95ed6219c115 100644 --- a/ci-operator/step-registry/storage/create/csi-gcp-filestore/storage-create-csi-gcp-filestore-ref.yaml +++ b/ci-operator/step-registry/storage/create/csi-gcp-filestore/storage-create-csi-gcp-filestore-ref.yaml @@ -1,8 +1,12 @@ ref: as: storage-create-csi-gcp-filestore - from: cli + from_image: + namespace: ocp + name: "4.15" + tag: upi-installer + # from: cli # inject oc binary - cli: latest + # cli: latest commands: storage-create-csi-gcp-filestore-commands.sh resources: requests: diff --git a/ci-operator/step-registry/storage/destroy/csi-gcp-filestore/storage-destroy-csi-gcp-filestore-commands.sh b/ci-operator/step-registry/storage/destroy/csi-gcp-filestore/storage-destroy-csi-gcp-filestore-commands.sh index af09b6fee585..90b21bf49386 100644 --- a/ci-operator/step-registry/storage/destroy/csi-gcp-filestore/storage-destroy-csi-gcp-filestore-commands.sh +++ b/ci-operator/step-registry/storage/destroy/csi-gcp-filestore/storage-destroy-csi-gcp-filestore-commands.sh @@ -6,6 +6,12 @@ set -o pipefail python3 --version export CLOUDSDK_PYTHON=python3 +if test -f "${SHARED_DIR}/proxy-conf.sh" +then + # shellcheck disable=SC1090 + source "${SHARED_DIR}/proxy-conf.sh" +fi + CLUSTER_ID="$(oc get -o jsonpath='{.status.infrastructureName}{"\n"}' infrastructure cluster)" export CLUSTER_ID