K8SPS-288 - async self healing test

Jenkinsfile

@@ -280,7 +280,6 @@ pipeline {
     environment {
         CLOUDSDK_CORE_DISABLE_PROMPTS = 1
         CLEAN_NAMESPACE = 1
-        OPERATOR_NS = 'ps-operator'
         GIT_SHORT_COMMIT = sh(script: 'git rev-parse --short HEAD', , returnStdout: true).trim()
         VERSION = "${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}"
         CLUSTER_NAME = sh(script: "echo jen-ps-${env.CHANGE_ID}-${GIT_SHORT_COMMIT}-${env.BUILD_NUMBER} | tr '[:upper:]' '[:lower:]'", , returnStdout: true).trim()

e2e-tests/run-distro.csv

@@ -21,6 +21,7 @@ one-pod
 operator-self-healing
 recreate
 scaling
+self-healing
 service-per-pod
 sidecars
 smart-update

e2e-tests/run-pr.csv

@@ -25,6 +25,7 @@ one-pod
 operator-self-healing
 recreate
 scaling
+self-healing
 service-per-pod
 sidecars
 smart-update

e2e-tests/run-release.csv

@@ -25,6 +25,7 @@ one-pod
 operator-self-healing
 recreate
 scaling
+self-healing
 service-per-pod
 sidecars
 smart-update

e2e-tests/tests/gr-self-healing/18-drop-finalizer.yaml

@@ -0,0 +1,5 @@
+apiVersion: ps.percona.com/v1alpha1
+kind: PerconaServerMySQL
+metadata:
+  name: gr-self-healing
+  finalizers: []

e2e-tests/tests/operator-self-healing/02-assert.yaml

@@ -58,16 +58,3 @@ status:
     size: 3
     state: ready
   state: ready
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: chaos-daemon
-status:
-  currentNumberScheduled: 3
-  desiredNumberScheduled: 3
-  numberAvailable: 3
-  numberMisscheduled: 0
-  numberReady: 3
-  observedGeneration: 1
-  updatedNumberScheduled: 3

e2e-tests/tests/operator-self-healing/12-drop-finalizer.yaml

@@ -0,0 +1,5 @@
+apiVersion: ps.percona.com/v1alpha1
+kind: PerconaServerMySQL
+metadata:
+  name: operator-self-healing
+  finalizers: []

e2e-tests/tests/self-healing/00-assert.yaml

@@ -0,0 +1,26 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: perconaservermysqls.ps.percona.com
+spec:
+  group: ps.percona.com
+  names:
+    kind: PerconaServerMySQL
+    listKind: PerconaServerMySQLList
+    plural: perconaservermysqls
+    shortNames:
+    - ps
+    singular: perconaservermysql
+  scope: Namespaced
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: check-operator-deploy-status
+timeout: 120
+commands:
+  - script: kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1

e2e-tests/tests/self-healing/00-deploy-operator.yaml

@@ -0,0 +1,14 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      deploy_operator
+      deploy_non_tls_cluster_secrets
+      deploy_tls_cluster_secrets
+      deploy_client

e2e-tests/tests/self-healing/01-assert.yaml

@@ -0,0 +1,27 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: chaos-controller-manager
+spec:
+  replicas: 3
+status:
+  availableReplicas: 3
+  readyReplicas: 3
+  replicas: 3
+  updatedReplicas: 3
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: chaos-daemon
+status:
+  currentNumberScheduled: 3
+  desiredNumberScheduled: 3
+  numberAvailable: 3
+  numberMisscheduled: 0
+  numberReady: 3
+  updatedNumberScheduled: 3

e2e-tests/tests/self-healing/01-deploy-chaos-mesh.yaml

@@ -0,0 +1,11 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      deploy_chaos_mesh
+    timeout: 120

e2e-tests/tests/self-healing/02-assert.yaml

@@ -0,0 +1,60 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 420
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: self-healing-mysql
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: self-healing-orc
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: self-healing-haproxy
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+apiVersion: ps.percona.com/v1alpha1
+kind: PerconaServerMySQL
+metadata:
+  name: self-healing
+  finalizers:
+    - percona.com/delete-mysql-pods-in-order
+status:
+  haproxy:
+    ready: 3
+    size: 3
+    state: ready
+  mysql:
+    ready: 3
+    size: 3
+    state: ready
+  orchestrator:
+    ready: 3
+    size: 3
+    state: ready
+  state: ready

e2e-tests/tests/self-healing/02-create-cluster.yaml

@@ -0,0 +1,21 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      get_cr \
+        | yq eval '.spec.mysql.clusterType="async"' - \
+        | yq eval '.spec.mysql.size=3' - \
+        | yq eval '.spec.mysql.affinity.antiAffinityTopologyKey="none"' - \
+        | yq eval '.spec.proxy.haproxy.enabled=true' - \
+        | yq eval '.spec.proxy.haproxy.size=3' - \
+        | yq eval '.spec.proxy.haproxy.affinity.antiAffinityTopologyKey="none"' - \
+        | yq eval '.spec.orchestrator.enabled=true' - \
+        | yq eval '.spec.orchestrator.size=3' - \
+        | yq eval '.spec.orchestrator.affinity.antiAffinityTopologyKey="none"' - \
+        | kubectl -n "${NAMESPACE}" apply -f -

e2e-tests/tests/self-healing/03-write-data.yaml

@@ -0,0 +1,16 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      run_mysql \
+      	"CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \
+      	"-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password"
+
+      run_mysql \
+      	"INSERT myDB.myTable (id) VALUES (100500)" \
+      	"-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password"

e2e-tests/tests/self-healing/04-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: 04-read-from-primary
+data:
+  data: "100500"

e2e-tests/tests/self-healing/04-read-from-primary.yaml

@@ -0,0 +1,13 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 30
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      data=$(run_mysql "SELECT * FROM myDB.myTable" "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password")
+
+      kubectl create configmap -n "${NAMESPACE}" 04-read-from-primary --from-literal=data="${data}"

e2e-tests/tests/self-healing/05-assert.yaml

@@ -0,0 +1,79 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: self-healing-mysql
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: self-healing-orc
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: self-healing-haproxy
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+apiVersion: ps.percona.com/v1alpha1
+kind: PerconaServerMySQL
+metadata:
+  name: self-healing
+  finalizers:
+    - percona.com/delete-mysql-pods-in-order
+status:
+  haproxy:
+    ready: 3
+    size: 3
+    state: ready
+  mysql:
+    ready: 3
+    size: 3
+    state: ready
+  orchestrator:
+    ready: 3
+    size: 3
+    state: ready
+  state: ready
+---
+apiVersion: chaos-mesh.org/v1alpha1
+kind: PodChaos
+metadata:
+  name: chaos-pod-kill-primary
+spec:
+  action: pod-kill
+  mode: one
+status:
+  experiment:
+    containerRecords:
+    - events:
+      - operation: Apply
+        type: Succeeded
+      injectedCount: 1
+      phase: Injected
+      recoveredCount: 0
+      selectorKey: .
+    desiredPhase: Run

e2e-tests/tests/self-healing/05-kill-primary.yaml

@@ -0,0 +1,18 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 30
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      init_pod="$(get_primary_from_haproxy ${test_name}-haproxy-0)"
+      kill_pods "${NAMESPACE}" "pod" "$init_pod" "" "primary"
+      sleep 10 # wait a bit for pod to be killed
+
+      if [ "$init_pod" == "$(get_primary_from_haproxy ${test_name}-haproxy-0)" ]; then
+        echo "primary pod was not killed! something went wrong."
+        exit 1
+      fi

e2e-tests/tests/self-healing/06-write-data.yaml

@@ -0,0 +1,12 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      run_mysql \
+      	"INSERT myDB.myTable (id) VALUES (100501)" \
+      	"-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password"