ci: add pipeline for storage network

Signed-off-by: Yang Chiu <yang.chiu@suse.com>

pipelines/storage_network/Dockerfile.setup

@@ -0,0 +1,31 @@
+From alpine:latest
+
+ARG KUBECTL_VERSION=v1.20.2
+
+ARG RKE_VERSION=v1.3.4
+
+ARG TERRAFORM_VERSION=1.3.5
+
+ARG YQ_VERSION=v4.24.2
+
+ENV WORKSPACE /src/longhorn-tests
+
+WORKDIR $WORKSPACE
+
+RUN wget -q https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl && \
+    mv kubectl /usr/local/bin/kubectl && \
+    chmod +x /usr/local/bin/kubectl && \
+    wget -q https://github.com/rancher/rke/releases/download/$RKE_VERSION/rke_linux-amd64 && \
+    mv rke_linux-amd64 /usr/bin/rke && \
+    chmod +x /usr/bin/rke && \
+    wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
+    unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip && rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
+    mv terraform /usr/bin/terraform && \
+    chmod +x /usr/bin/terraform && \
+    wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" && \
+    mv yq_linux_amd64 /usr/local/bin/yq && \
+    chmod +x /usr/local/bin/yq && \
+    apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip aws-cli && \
+    ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa
+
+COPY [".", "$WORKSPACE"]

pipelines/storage_network/Jenkinsfile

@@ -0,0 +1,218 @@
+def imageName = "${JOB_BASE_NAME}-${env.BUILD_NUMBER}"
+def summary
+def WORKSPACE = "/src/longhorn-tests"
+def BUILD_TRIGGER_BY = "\n${currentBuild.getBuildCauses()[0].shortDescription}"
+
+// define optional parameters
+def SELINUX_MODE = params.SELINUX_MODE ? params.SELINUX_MODE : ""
+
+def CREDS_ID = JOB_BASE_NAME == "longhorn-tests-regression" ? "AWS_CREDS_RANCHER_QA" : "AWS_CREDS"
+def REGISTRATION_CODE_ID = params.ARCH == "amd64" ? "REGISTRATION_CODE" : "REGISTRATION_CODE_ARM64"
+
+// parameters for air gap installation
+def AIR_GAP_INSTALLATION = params.AIR_GAP_INSTALLATION ? params.AIR_GAP_INSTALLATION : false
+def LONGHORN_INSTALL_VERSION = params.LONGHORN_INSTALL_VERSION ? params.LONGHORN_INSTALL_VERSION : "master"
+def LONGHORN_TRANSIENT_VERSION = params.LONGHORN_TRANSIENT_VERSION ? params.LONGHORN_TRANSIENT_VERSION : ""
+def CIS_HARDENING = params.CIS_HARDENING ? params.CIS_HARDENING : false
+def REGISTRY_URL
+def REGISTRY_USERNAME
+def REGISTRY_PASSWORD
+
+// parameter for hdd test
+def USE_HDD = params.USE_HDD ? params.USE_HDD : false
+
+node {
+
+    withCredentials([
+        usernamePassword(credentialsId: CREDS_ID, passwordVariable: 'AWS_SECRET_KEY', usernameVariable: 'AWS_ACCESS_KEY'),
+        string(credentialsId: 'DO_CREDS', variable: 'DO_TOKEN'),
+        string(credentialsId: REGISTRATION_CODE_ID, variable: 'REGISTRATION_CODE'),
+    ]) {
+
+        if (params.SEND_SLACK_NOTIFICATION) {
+            notifyBuild('STARTED', BUILD_TRIGGER_BY, params.NOTIFY_SLACK_CHANNEL)
+        }
+
+        checkout scm
+
+        try {
+
+            if (params.AIR_GAP_INSTALLATION) {
+
+                stage('airgap build') {
+                    sh "airgap/scripts/build.sh"
+                    sh """ docker run -itd --name airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} \
+                                           --env TF_VAR_longhorn_version=${LONGHORN_INSTALL_VERSION} \
+                                           --env TF_VAR_do_token=${DO_TOKEN} \
+                                           --env TF_VAR_aws_access_key=${AWS_ACCESS_KEY} \
+                                           --env TF_VAR_aws_secret_key=${AWS_SECRET_KEY} \
+                                           airgap-${JOB_BASE_NAME}-${BUILD_NUMBER}
+                    """
+                }
+
+                stage ('airgap setup') {
+                    sh "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} ./airgap/scripts/terraform-setup.sh"
+                    REGISTRY_URL = sh (
+                        script: "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} terraform -chdir=./airgap/terraform output -raw registry_url",
+                        returnStdout: true
+                    )
+                    println REGISTRY_URL
+                    REGISTRY_USERNAME = sh (
+                        script: "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} terraform -chdir=./airgap/terraform output -raw registry_username",
+                        returnStdout: true
+                    )
+                    REGISTRY_PASSWORD = sh (
+                        script: "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} terraform -chdir=./airgap/terraform output -raw registry_password",
+                        returnStdout: true
+                    )
+                }
+
+            }
+
+            stage('build') {
+
+                echo "Using credentials: $CREDS_ID"
+                echo "Using registration code: $REGISTRATION_CODE_ID"
+
+                sh "pipelines/storage_network/scripts/build.sh"
+                sh """ docker run -itd --name ${JOB_BASE_NAME}-${BUILD_NUMBER} \
+                                       --env AIR_GAP_INSTALLATION=${AIR_GAP_INSTALLATION} \
+                                       --env REGISTRY_URL=${REGISTRY_URL} \
+                                       --env REGISTRY_USERNAME=${REGISTRY_USERNAME} \
+                                       --env REGISTRY_PASSWORD=${REGISTRY_PASSWORD} \
+                                       --env LONGHORN_INSTALL_VERSION=${LONGHORN_INSTALL_VERSION} \
+                                       --env CUSTOM_LONGHORN_ENGINE_IMAGE=${CUSTOM_LONGHORN_ENGINE_IMAGE} \
+                                       --env CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE=${CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE} \
+                                       --env CUSTOM_LONGHORN_MANAGER_IMAGE=${CUSTOM_LONGHORN_MANAGER_IMAGE} \
+                                       --env CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE=${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE} \
+                                       --env CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE=${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE} \
+                                       --env LONGHORN_TESTS_CUSTOM_IMAGE=${LONGHORN_TESTS_CUSTOM_IMAGE} \
+                                       --env DISTRO=${DISTRO} \
+                                       --env LONGHORN_REPO_URI=${LONGHORN_REPO_URI} \
+                                       --env LONGHORN_REPO_BRANCH=${LONGHORN_REPO_BRANCH} \
+                                       --env LONGHORN_STABLE_VERSION=${LONGHORN_STABLE_VERSION} \
+                                       --env LONGHORN_TRANSIENT_VERSION=${LONGHORN_TRANSIENT_VERSION} \
+                                       --env LONGHORN_TEST_CLOUDPROVIDER=${LONGHORN_TEST_CLOUDPROVIDER} \
+                                       --env LONGHORN_UPGRADE_TEST=${LONGHORN_UPGRADE_TEST} \
+                                       --env PYTEST_CUSTOM_OPTIONS="${PYTEST_CUSTOM_OPTIONS}" \
+                                       --env BACKUP_STORE_TYPE="${BACKUP_STORE_TYPE}" \
+                                       --env TF_VAR_use_hdd=${USE_HDD} \
+                                       --env TF_VAR_arch=${ARCH} \
+                                       --env TF_VAR_k8s_distro_name=${K8S_DISTRO_NAME} \
+                                       --env TF_VAR_k8s_distro_version=${K8S_DISTRO_VERSION} \
+                                       --env TF_VAR_aws_availability_zone=${AWS_AVAILABILITY_ZONE} \
+                                       --env TF_VAR_aws_region=${AWS_REGION} \
+                                       --env TF_VAR_os_distro_version=${DISTRO_VERSION} \
+                                       --env TF_VAR_do_token=${env.TF_VAR_do_token} \
+                                       --env TF_VAR_aws_access_key=${AWS_ACCESS_KEY} \
+                                       --env TF_VAR_lh_aws_instance_name_controlplane="${JOB_BASE_NAME}-ctrl" \
+                                       --env TF_VAR_lh_aws_instance_name_worker="${JOB_BASE_NAME}-wrk" \
+                                       --env TF_VAR_lh_aws_instance_type_controlplane=${CONTROLPLANE_INSTANCE_TYPE} \
+                                       --env TF_VAR_lh_aws_instance_type_worker=${WORKER_INSTANCE_TYPE}\
+                                       --env TF_VAR_aws_secret_key=${AWS_SECRET_KEY} \
+                                       --env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY} \
+                                       --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_KEY} \
+                                       --env AWS_DEFAULT_REGION=${AWS_REGION} \
+                                       --env TF_VAR_selinux_mode=${SELINUX_MODE} \
+                                       --env TF_VAR_registration_code=${REGISTRATION_CODE} \
+                                       --env TF_VAR_cis_hardening=${CIS_HARDENING} \
+                                       --env TF_VAR_mtu=${MTU_SIZE} \
+                                       --env TF_VAR_multus_version=${MULTUS_VERSION} \
+                                       --env TF_VAR_thick_plugin=${THICK_PLUGIN} \
+                                       ${imageName}
+                """
+            }
+
+            timeout(60) {
+                stage ('terraform') {
+                    sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/storage_network/scripts/terraform-setup.sh"
+                }
+			}
+
+            stage ('longhorn setup & tests') {
+                sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/storage_network/scripts/longhorn-setup.sh"
+            }
+
+            stage ('download support bundle') {
+                sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/storage_network/scripts/download-support-bundle.sh  ${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip"
+                sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip ."
+				archiveArtifacts allowEmptyArchive: true, artifacts: '**/*.zip', followSymlinks: false
+			}
+
+            stage ('report generation') {
+                sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-junit-report.xml ."
+
+                if(params.LONGHORN_UPGRADE_TEST && params.LONGHORN_TRANSIENT_VERSION) {
+                    sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-stable-junit-report.xml ."
+                    sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-transient-junit-report.xml ."
+                    summary = junit 'longhorn-test-upgrade-from-stable-junit-report.xml, longhorn-test-upgrade-from-transient-junit-report.xml, longhorn-test-junit-report.xml'
+                }
+                else if(params.LONGHORN_UPGRADE_TEST) {
+                    sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-stable-junit-report.xml ."
+                    summary = junit 'longhorn-test-upgrade-from-stable-junit-report.xml, longhorn-test-junit-report.xml'
+                }
+                else {
+                    summary = junit 'longhorn-test-junit-report.xml'
+                }
+            }
+
+        } catch (e) {
+            currentBuild.result = "FAILED"
+            throw e
+        } finally {
+            stage ('releasing resources') {
+                if (sh (script: "docker container inspect airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} > /dev/null 2>&1", returnStatus: true) == 0) {
+                    sh "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} ./airgap/scripts/cleanup.sh"
+                    sh "docker stop airgap-${JOB_BASE_NAME}-${BUILD_NUMBER}"
+                    sh "docker rm -v airgap-${JOB_BASE_NAME}-${BUILD_NUMBER}"
+                    sh "docker rmi airgap-${JOB_BASE_NAME}-${BUILD_NUMBER}"
+                }
+
+                if (sh (script: "docker container inspect ${JOB_BASE_NAME}-${BUILD_NUMBER} > /dev/null 2>&1", returnStatus: true) == 0) {
+                    sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/storage_network/scripts/cleanup.sh"
+                    sh "docker stop ${JOB_BASE_NAME}-${BUILD_NUMBER}"
+                    sh "docker rm -v ${JOB_BASE_NAME}-${BUILD_NUMBER}"
+                    sh "docker rmi ${imageName}"
+                }
+
+                if (summary) {
+                    summary_msg = "\nTest Summary - Failures: ${summary.failCount}, Skipped: ${summary.skipCount}, Passed: ${summary.passCount}  -- Job completed in ${currentBuild.durationString.replace(' and counting', '')}"
+                } else {
+                    summary_msg = "\n Test Failed: No Junit report"
+                }
+
+                if(params.SEND_SLACK_NOTIFICATION){
+                    notifyBuild(currentBuild.result, summary_msg, params.NOTIFY_SLACK_CHANNEL)
+                }
+            }
+        }
+    }
+
+}
+
+
+def notifyBuild(String buildStatus = 'STARTED', String summary_msg, String slack_channel) {
+  // build status of null means successful
+  buildStatus =  buildStatus ?: 'SUCCESSFUL'
+
+  // Default values
+  def colorName = 'RED'
+  def colorCode = '#FF0000'
+  def subject = "${buildStatus}: Job '${env.JOB_BASE_NAME} [${env.BUILD_NUMBER}]'"
+  def summary = "${subject} (${env.BUILD_URL})" + summary_msg
+
+  // Override default values based on build status
+  if (buildStatus == 'STARTED') {
+    color = 'YELLOW'
+    colorCode = '#FFFF00'
+  } else if (buildStatus == 'SUCCESSFUL') {
+    color = 'GREEN'
+    colorCode = '#00FF00'
+  } else {
+    color = 'RED'
+    colorCode = '#FF0000'
+  }
+
+  // Send notifications
+  slackSend (color: colorCode, message: summary, channel: slack_channel,  tokenCredentialId: 'longhorn-tests-slack-token')
+}

pipelines/storage_network/scripts/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build --no-cache -f ./pipelines/storage_network/Dockerfile.setup -t "${JOB_BASE_NAME}-${BUILD_NUMBER}" .

pipelines/storage_network/scripts/cleanup.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# terminate any terraform processes
+TERRAFORM_PIDS=( `ps aux | grep -i terraform | grep -v grep | awk '{printf("%s ",$1)}'` )
+if [[ -n ${TERRAFORM_PIDS[@]} ]] ; then
+	for PID in "${TERRAFORM_PIDS[@]}"; do
+		kill "${TERRAFORM_PIDS}"
+	done
+fi
+
+# wait 30 seconds for graceful terraform termination
+sleep 30
+
+terraform -chdir=pipelines/storage_network/terraform destroy -auto-approve -no-color

pipelines/storage_network/scripts/download-support-bundle.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -e
+
+SUPPORT_BUNDLE_FILE_NAME=${1:-"lh-support-bundle.zip"}
+SUPPORT_BUNDLE_ISSUE_URL=${2:-""}
+SUPPORT_BUNDLE_ISSUE_DESC=${3:-"Auto-generated support bundle"}
+
+set_kubeconfig_envvar(){
+    export KUBECONFIG="${PWD}/pipelines/storage_network/terraform/k3s.yaml"
+}
+
+set_kubeconfig_envvar
+
+LH_FRONTEND_ADDR=`kubectl get svc -n longhorn-system longhorn-frontend -o json | jq -r '.spec.clusterIP + ":" + (.spec.ports[0].port|tostring)'`
+
+JSON_PAYLOAD="{\"issueURL\": \"${SUPPORT_BUNDLE_ISSUE_DESC}\", \"description\": \"${SUPPORT_BUNDLE_ISSUE_DESC}\"}"
+
+CURL_CMD="curl -XPOST http://${LH_FRONTEND_ADDR}/v1/supportbundles -H 'Accept: application/json' -H 'Accept-Encoding: gzip, deflate' -d '"${JSON_PAYLOAD}"'"
+
+SUPPORT_BUNDLE_URL=`kubectl exec -n longhorn-system svc/longhorn-frontend -- bash -c "${CURL_CMD}"  | jq -r '.links.self + "/" + .name'`
+
+SUPPORT_BUNDLE_READY=false
+while [[ ${SUPPORT_BUNDLE_READY} == false ]]; do
+    PERCENT=`kubectl exec -n longhorn-system svc/longhorn-frontend -- curl -H 'Accept: application/json' ${SUPPORT_BUNDLE_URL} | jq -r '.progressPercentage' || true`
+    echo ${PERCENT}
+
+    if [[ ${PERCENT} == 100 ]]; then SUPPORT_BUNDLE_READY=true; fi
+done
+
+kubectl exec -n longhorn-system svc/longhorn-frontend -- curl -H 'Accept-Encoding: gzip, deflate' ${SUPPORT_BUNDLE_URL}/download > ${SUPPORT_BUNDLE_FILE_NAME}

pipelines/storage_network/scripts/longhorn-setup.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+set -x
+
+source pipelines/utilities/selinux_workaround.sh
+source pipelines/utilities/install_csi_snapshotter.sh
+source pipelines/utilities/create_aws_secret.sh
+source pipelines/utilities/install_backupstores.sh
+source pipelines/utilities/storage_network.sh
+source pipelines/utilities/create_longhorn_namespace.sh
+source pipelines/utilities/longhorn_manifest.sh
+source pipelines/utilities/run_longhorn_test.sh
+
+# create and clean tmpdir
+TMPDIR="/tmp/longhorn"
+mkdir -p ${TMPDIR}
+rm -rf "${TMPDIR}/"
+
+export LONGHORN_NAMESPACE="longhorn-system"
+export LONGHORN_INSTALL_METHOD="manifest"
+
+set_kubeconfig_envvar(){
+    export KUBECONFIG="${PWD}/pipelines/storage_network/terraform/k3s.yaml"
+}
+
+main(){
+  set_kubeconfig_envvar
+
+  if [[ ${DISTRO} == "rhel" ]] || [[ ${DISTRO} == "rockylinux" ]] || [[ ${DISTRO} == "oracle" ]]; then
+    apply_selinux_workaround
+  fi
+
+  # set debugging mode off to avoid leaking aws secrets to the logs.
+  # DON'T REMOVE!
+  set +x
+  create_aws_secret
+  set -x
+
+  if [[ "${TF_VAR_thick_plugin}" == true ]]; then
+    deploy_multus_thick_plugin_daemonset
+  else
+    deploy_multus_thin_plugin_daemonset
+  fi
+  deploy_network_attachment_definition
+
+  create_longhorn_namespace
+  install_backupstores
+  install_csi_snapshotter
+
+  generate_longhorn_yaml_manifest
+  install_longhorn_by_manifest
+
+  update_storage_network_setting
+
+  run_longhorn_test
+}
+
+main

pipelines/storage_network/scripts/terraform-setup.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -x
+
+terraform -chdir=pipelines/storage_network/terraform init
+terraform -chdir=pipelines/storage_network/terraform apply -auto-approve -no-color
+
+NETWORK_INTERFACE_IDS=$(terraform -chdir=pipelines/storage_network/terraform output -json network_interface_ids | tr -d '"')
+for id in ${NETWORK_INTERFACE_IDS}; do
+  aws ec2 modify-network-interface-attribute --network-interface-id "${id}" --no-source-dest-check
+done
+
+exit $?