chore(test): Add Weekly Testgrid Tests for Common kURL Add-on Migrati…

…ons (#4697)

* Add weekly kURL migration spec test suite

* add multi-node test for customer

* testgrid does not support this type of multi-node test

* Add more description to test

* Add validation after the upgrade

* verify data migration post upgrade

.github/workflows/cron-testgrid-weekly.yaml

@@ -25,3 +25,23 @@ jobs:
             --spec ./testgrid/specs/full.yaml \
             --os-spec ./testgrid/specs/os-firstlast.yaml \
             --priority -2
+  testgrid-weekly-customer-migration-specs:
+    if: ${{ github.repository_owner == 'replicatedhq' }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: pull tgrun image
+      run: docker pull replicated/tgrun:latest
+
+    - name: testgrid-queue-prod
+      env:
+        TESTGRID_API_TOKEN: ${{ secrets.TESTGRID_PROD_API_TOKEN }}
+      run: |
+        docker run --rm -e TESTGRID_API_TOKEN -v `pwd`:/wrk -w /wrk \
+          replicated/tgrun:latest queue \
+            --ref "PROD-weekly-customer-migration-specs-$(date --utc +%FT%TZ)" \
+            --spec ./testgrid/specs/customer-migration-specs.yaml \
+            --os-spec ./testgrid/specs/customer-common-os.yaml \
+            --priority -2

testgrid/specs/customer-common-os.yaml

@@ -0,0 +1,11 @@
+- id: centos-84
+  name: CentOS
+  version: "8.4"
+  vmimageuri: https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2
+  preinit: ""
+- id: ubuntu-2204
+  name: Ubuntu
+  version: "22.04"
+  vmimageuri: https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img
+  preinit: ""
+

testgrid/specs/customer-migration-specs.yaml

@@ -0,0 +1,75 @@
+- name: "Cust sha1:171aaa33103c04449567bfeadb0a49833ca7a6b0: k8s 1.25.4 -> 1.27, containerd 1.6.19 -> 1.6.x, weave -> flannel, OpenEBS 3.4.x -> 3.7.x"
+  flags: "yes"
+  installerSpec:
+    kubernetes:
+      version: 1.25.4
+    containerd:
+      version: 1.6.19
+    weave:
+      version: 2.6.x
+    kotsadm:
+      version: 1.96.3
+      disableS3: true
+    ekco:
+      version: 0.26.4
+    openebs:
+      version: 3.4.x
+      isLocalPVEnabled: true
+      localPVStorageClassName: local
+    prometheus:
+      version: "0.63.0-45.7.1"
+    metricsServer:
+      version: 0.6.3
+  upgradeSpec:
+    kubernetes:
+      version: 1.27.x
+    containerd:
+      version: 1.6.x
+    flannel:
+      version: 0.22.x
+    kotsadm:
+      version: 1.100.x
+      disableS3: true
+    ekco:
+      version: latest
+    openebs:
+      version: 3.7.x
+      isLocalPVEnabled: true
+      localPVStorageClassName: local
+    prometheus:
+      version: 0.65.x
+    metricsServer:
+      version: 0.6.x
+  postInstallScript: |
+    # source helper functions
+    source /opt/kurl-testgrid/testhelpers.sh
+
+    # mount a localpv volume
+    create_deployment_with_mounted_volume "migration-test" "default" "/data" "registry:2.8.1"
+
+    # write to the volume
+    create_random_file_and_upload_to_deployment "migration-test" "default" "./test.data" "/data/test.data"
+
+    # deploy sample app 
+    install_and_customize_kurl_integration_test_application
+  postUpgradeScript: |
+    # source helper functions
+    source /opt/kurl-testgrid/testhelpers.sh
+
+    # verify data was migrated
+    download_file_from_deployment_and_compare "migration-test" "default" "./test.data" "/data/test.data"
+
+    # ensure we can redeploy sample app
+    check_and_customize_kurl_integration_test_application
+
+    # ensure weave was removed
+    if ls /etc/cni/net.d/*weave* >/dev/null 2>&1; then
+      echo "weave CNI still installed on host"
+      exit 1
+    fi
+
+    # ensure kubernetes was upgraded
+    if [[ ! "$(kubectl version --short | grep -i server | awk '{ print $3 }' | sed 's/^v*//')" =~ "1.27" ]]; then
+      echo "Kubernetes was not upgraded to 1.27.x"
+      exit 1
+    fi