Merge branch 'w/2.9/bugfix/ZENKO-4903-fix-drctl-status-flaky' into tm…

…p/octopus/w/2.10/bugfix/ZENKO-4903-fix-drctl-status-flaky
scality · Oct 7, 2024 · 0d337c5 · 0d337c5
2 parents 85a3c36 + 4ebdcec
commit 0d337c5
Show file tree

Hide file tree

Showing 7 changed files with 52 additions and 43 deletions.
diff --git a/.github/scripts/end2end/bootstrap-kind.sh b/.github/scripts/end2end/bootstrap-kind.sh
@@ -2,7 +2,7 @@
 
 set -exu
 
-NODE_IMAGE=${1:-kindest/node:v1.26.6@sha256:6e2d8b28a5b601defe327b98bd1c2d1930b49e5d8c512e1895099e4504007adb}
+NODE_IMAGE=${1:-kindest/node:v1.27.16@sha256:3fd82731af34efe19cd54ea5c25e882985bafa2c9baefe14f8deab1737d9fabe}
 VOLUME_ROOT=${2:-/artifacts}
 WORKER_NODE_COUNT=${3:-0}
 CLUSTER_NAME=${CLUSTER_NAME:-kind}
@@ -36,6 +36,12 @@ add_workers() {
         count=$((count+1))
         echo "- role: worker
   image: ${NODE_IMAGE}
+  kubeadmConfigPatches:
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: 'ingress-ready=true'
   extraMounts:
   - hostPath: ${VOLUME_ROOT}/data
     containerPath: /data
@@ -61,6 +67,9 @@ nodes:
     nodeRegistration:
       kubeletExtraArgs:
         node-labels: "ingress-ready=true"
+  - |
+    kind: KubeletConfiguration
+    cgroupDriver: systemd
   extraMounts:
   - hostPath: ${VOLUME_ROOT}/data
     containerPath: /data

diff --git a/.github/scripts/end2end/install-kind-dependencies.sh b/.github/scripts/end2end/install-kind-dependencies.sh
@@ -13,7 +13,7 @@ source "${VERSION_FILE}"
 ZK_OPERATOR_VERSION=0.2.15
 CERT_MANAGER_VERSION=v1.6.1
 KAFKA_OPERATOR_VERSION=0.25.1
-INGRESS_NGINX_VERSION=controller-v1.8.1
+INGRESS_NGINX_VERSION=controller-v1.11.2
 PROMETHEUS_VERSION=v0.52.1
 KEYCLOAK_VERSION=18.4.4
 

diff --git a/.github/workflows/end2end.yaml b/.github/workflows/end2end.yaml
@@ -16,7 +16,7 @@ env:
   WORKER_COUNT: '2'
   OPERATOR_REPO: git@github.com:scality/zenko-operator.git
   OPERATOR_IMAGE: ""
-  KIND_NODE_IMAGE: "kindest/node:v1.23.4@sha256:0e34f0d0fd448aa2f2819cfd74e99fe5793a6e4938b328f657c8e3f81ee0dfb9"
+  KIND_NODE_IMAGE: "kindest/node:v1.27.16@sha256:3fd82731af34efe19cd54ea5c25e882985bafa2c9baefe14f8deab1737d9fabe"
   VOLUME_ROOT: /artifacts
   OIDC_REALM: "zenko"
   OIDC_CLIENT_ID: "zenko-ui"
@@ -390,9 +390,7 @@ jobs:
   end2end-http:
     needs: [build-kafka, build-test-image, check-dashboard-versions]
     runs-on:
-      - ubuntu
-      - focal
-      - xlarge
+      - ubuntu-22.04-8core
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -533,9 +531,7 @@ jobs:
   end2end-https:
     needs: [build-kafka, build-test-image, check-dashboard-versions]
     runs-on:
-      - ubuntu
-      - focal
-      - xlarge
+      - ubuntu-22.04-8core
     env:
       UI_ENDPOINT: 'https://ui.zenko.local'
       OIDC_ENDPOINT: 'https://keycloak.zenko.local'

diff --git a/solution/deps.yaml b/solution/deps.yaml
@@ -21,7 +21,7 @@ cloudserver:
 drctl:
   sourceRegistry: ghcr.io/scality
   image: zenko-drctl
-  tag: v1.0.4
+  tag: e43bce2bfddd5c667e1488c2a2fa846da53c8c07
   envsubst: DRCTL_TAG
   toolName: zenko-drctl
 fubectl:

diff --git a/tests/ctst/steps/dr/drctl.ts b/tests/ctst/steps/dr/drctl.ts
@@ -108,32 +108,30 @@ type FailoverConfig = {
     timeout?: string;
     sinkKubeconfigPath?: string;
     sinkKubeconfigData?: string;
-    sinkZenkoInstance?: string;
-    sinkZenkoNamespace?: string;
+    sinkZenkoDrInstance?: string;
+    sinkZenkoDrNamespace?: string;
 };
 
 type FailbackConfig = {
     wait?: boolean;
     timeout?: string;
     sinkKubeconfigPath?: string;
     sinkKubeconfigData?: string;
-    sinkZenkoInstance?: string;
-    sinkZenkoNamespace?: string;
+    sinkZenkoDrInstance?: string;
+    sinkZenkoDrNamespace?: string;
 };
 
 type UninstallConfig = {
-    sinkZenkoDrInstance?: string;
-    sourceZenkoDrInstance?: string;
     wait?: boolean;
     timeout?: string;
     sourceKubeconfigPath?: string;
     sourceKubeconfigData?: string;
     sinkKubeconfigPath?: string;
     sinkKubeconfigData?: string;
-    sinkZenkoInstance?: string;
-    sinkZenkoNamespace?: string;
-    sourceZenkoInstance?: string;
-    sourceZenkoNamespace?: string;
+    sinkZenkoDrInstance?: string;
+    sinkZenkoDrNamespace?: string;
+    sourceZenkoDrInstance?: string;
+    sourceZenkoDrNamespace?: string;
 };
 
 type StatusConfig = {
@@ -148,17 +146,16 @@ type StatusConfig = {
     sourceZenkoDrInstance?: string;
     sinkZenkoDrInstance?: string;
     output?: string;
+    outputFormat?: string;
 };
 
 type ReplicationPauseConfig = {
     sourceKubeconfigPath?: string;
     sourceKubeconfigData?: string;
     sinkKubeconfigPath?: string;
     sinkKubeconfigData?: string;
-    sourceZenkoInstance?: string;
-    sourceZenkoNamespace?: string;
-    sinkZenkoInstance?: string;
-    sinkZenkoNamespace?: string;
+    sourceZenkoDrNamespace?: string;
+    sinkZenkoDrNamespace?: string;
     sourceZenkoDrInstance?: string;
     sinkZenkoDrInstance?: string;
     wait?: boolean;
@@ -170,10 +167,8 @@ type ReplicationResumeConfig = {
     sourceKubeconfigData?: string;
     sinkKubeconfigPath?: string;
     sinkKubeconfigData?: string;
-    sourceZenkoInstance?: string;
-    sourceZenkoNamespace?: string;
-    sinkZenkoInstance?: string;
-    sinkZenkoNamespace?: string;
+    sourceZenkoDrNamespace?: string;
+    sinkZenkoDrNamespace?: string;
     sourceZenkoDrInstance?: string;
     sinkZenkoDrInstance?: string;
     wait?: boolean;

diff --git a/tests/ctst/steps/pra.ts b/tests/ctst/steps/pra.ts
@@ -110,15 +110,14 @@ async function waitForPhase(
             sourceZenkoNamespace: 'default',
             sinkZenkoDrInstance: 'end2end-pra-sink',
             sourceZenkoDrInstance: 'end2end-source',
-            output: 'json',
+            outputFormat: 'json',
         });
 
         if (!currentStatus) {
-            world.logger.debug('Failed to get DR status, retrying', {
+            world.logger.debug('Failed to get DR status', {
                 currentStatus,
             });
-            await Utils.sleep(1000);
-            continue;
+            throw new Error('Failed to get DR status');
         }
 
         const lines = currentStatus.split('\n');
@@ -217,7 +216,8 @@ Then('the DR sink should be in phase {string}', { timeout: 360000 }, async funct
         throw new Error(`Unknown state ${state}`);
     }
 
-    await waitForPhase(this, 'sink', targetPhase);
+    const res = await waitForPhase(this, 'sink', targetPhase);
+    assert(res);
 });
 
 Then('the DR source should be in phase {string}', { timeout: 360000 }, async function (this: Zenko, state: string) {
@@ -245,7 +245,8 @@ Then('the DR source should be in phase {string}', { timeout: 360000 }, async fun
         throw new Error(`Unknown state ${state}`);
     }
 
-    await waitForPhase(this, 'source', targetPhase);
+    const res = await waitForPhase(this, 'source', targetPhase);
+    assert(res);
 });
 
 Then('object {string} should {string} be {string} and have the storage class {string} on {string} site',
@@ -328,7 +329,8 @@ Then('the kafka DR volume exists', { timeout: volumeTimeout + 2000 }, async func
 const failoverTimeout = 360000;
 When ('I request the failover state for the DR', { timeout: failoverTimeout + 2000 }, async function (this: Zenko) {
     await this.zenkoDrCtl?.failover({
-        sinkZenkoNamespace: 'default',
+        sinkZenkoDrNamespace: 'default',
+        sinkZenkoDrInstance: 'end2end-pra-sink',
         wait: true,
         timeout: `${failoverTimeout.toString()}ms`,
     });
@@ -337,7 +339,8 @@ When ('I request the failover state for the DR', { timeout: failoverTimeout + 20
 const failbackTimeout = 360000;
 When ('I resume operations for the DR', { timeout: failbackTimeout + 2000 }, async function (this: Zenko) {
     await this.zenkoDrCtl?.failback({
-        sinkZenkoNamespace: 'default',
+        sinkZenkoDrNamespace: 'default',
+        sinkZenkoDrInstance: 'end2end-pra-sink',
         wait: true,
         timeout: `${failbackTimeout.toString()}ms`,
     });
@@ -348,8 +351,8 @@ When('I pause the DR', { timeout: pauseTimeout + 2000 }, async function (this: Z
     await this.zenkoDrCtl?.replicationPause({
         sourceZenkoDrInstance: 'end2end-source',
         sinkZenkoDrInstance: 'end2end-pra-sink',
-        sinkZenkoNamespace: 'default',
-        sourceZenkoNamespace: 'default',
+        sinkZenkoDrNamespace: 'default',
+        sourceZenkoDrNamespace: 'default',
         wait: true,
         timeout: `${pauseTimeout.toString()}ms`,
     });
@@ -360,10 +363,10 @@ When('I resume the DR', { timeout: resumeTimeout + 2000 }, async function (this:
     await this.zenkoDrCtl?.replicationResume({
         sourceZenkoDrInstance: 'end2end-source',
         sinkZenkoDrInstance: 'end2end-pra-sink',
-        sinkZenkoNamespace: 'default',
-        sourceZenkoNamespace: 'default',
+        sinkZenkoDrNamespace: 'default',
+        sourceZenkoDrNamespace: 'default',
         wait: true,
-        timeout: `${resumeTimeout.toString()}ms`,
+        timeout: `${pauseTimeout.toString()}ms`,
     });
 });
 
@@ -372,8 +375,8 @@ When('I uninstall DR', { timeout: uninstallTimeout + 2000 }, async function (thi
     await this.zenkoDrCtl?.uninstall({
         sourceZenkoDrInstance: 'end2end-source',
         sinkZenkoDrInstance: 'end2end-pra-sink',
-        sinkZenkoNamespace: 'default',
-        sourceZenkoNamespace: 'default',
+        sinkZenkoDrNamespace: 'default',
+        sourceZenkoDrNamespace: 'default',
         wait: true,
         timeout: `${uninstallTimeout.toString()}ms`,
     });

diff --git a/tests/ctst/steps/utils/utils.ts b/tests/ctst/steps/utils/utils.ts
@@ -297,9 +297,15 @@ async function verifyObjectLocation(this: Zenko, objectName: string,
         this.addCommandParameter({ versionId });
     }
     let conditionOk = false;
+
+    const startTime = Date.now();
+
     while (!conditionOk) {
         const res = await S3.headObject(this.getCommandParameters());
         if (res.err?.includes('NotFound')) {
+            if (Date.now() - startTime > 300000) {
+                throw new Error('Object not found after 300 seconds');
+            }
             await Utils.sleep(1000);
             continue;
         } else if (res.err) {