chore: run integration on multi node cluster

.github/workflows/integration.yml

@@ -10,6 +10,9 @@ on:
 jobs:
   integration:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: [ "5.0.1" ]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -22,11 +25,15 @@ jobs:
           go-version: '1.20'
 
       - uses: actions/checkout@v3
-      - name: Start containers
-        run: make compose
+      - name: Start cluster
+        run: make create-cluster version=${{ matrix.version }}
 
       - name: Integration Test
         run: go test -v test/integration/integration_test.go
         env:
           INPUT_PUBLISH: false
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/checkout@v3
+      - name: Delete cluster
+        run: make delete-cluster

Makefile

@@ -19,8 +19,11 @@ lint:
 test:
 	go test ./... -bench .
 
-compose:
-	docker compose up --wait --build --force-recreate --remove-orphans
+create-cluster:
+	bash scripts/create_cluster.sh --version $(version)
+
+delete-cluster:
+	bash scripts/delete_cluster.sh
 
 docker-build:
 	docker build --progress=plain -t docker.io/trendyoltech/dcp .

scripts/cluster.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+sh create_cluster.sh
+
+read -r -p "press enter to delete cluster _ "
+
+sh delete_cluster.sh

scripts/create_cluster.sh

@@ -0,0 +1,206 @@
+#!/bin/bash
+
+POSITIONAL_ARGS=()
+
+VERSION=7.2.4
+NAME_PREFIX="couchbase_node_"
+SUBNET="13.37.11.0/24"
+TOTAL_NODE=12
+REST_PORT=8091
+MEMORY_AS_MB=512
+USERNAME=user
+PASSWORD=123456
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -v|--version)
+      VERSION="$2"
+      shift
+      shift;;
+    -np|--name-prefix)
+      NAME_PREFIX="$2"
+      shift
+      shift;;
+    -s|--subnet)
+      SUBNET="$2"
+      shift
+      shift;;
+    -tn|--total-node)
+      TOTAL_NODE="$2"
+      shift
+      shift;;
+    -rp|--rest-port)
+      REST_PORT="$2"
+      shift
+      shift;;
+    -mam|--memory-as-mb)
+      MEMORY_AS_MB="$2"
+      shift
+      shift;;
+    -u|--username)
+      USERNAME="$2"
+      shift
+      shift;;
+    -p|--password)
+      PASSWORD="$2"
+      shift
+      shift;;
+    -*|--*)
+      echo "Unknown option $1"
+      exit 1;;
+    *)
+      POSITIONAL_ARGS+=("$1")
+      shift;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+echo "NAME_PREFIX = $NAME_PREFIX"
+echo "VERSION = $VERSION"
+echo "SUBNET = $SUBNET"
+echo "TOTAL_NODE = $TOTAL_NODE"
+echo "REST_PORT = $REST_PORT"
+echo "MEMORY_AS_MB = $MEMORY_AS_MB"
+echo "USERNAME = $USERNAME"
+echo "PASSWORD = $PASSWORD"
+echo "---------------------------"
+
+MASTER=""
+KNOWN_NODES=""
+
+echo "creating network"
+docker network create couchbase --subnet=$SUBNET
+
+check_db() {
+  URL="http://$1:$REST_PORT/pools"
+  curl --silent --max-time 5 "$URL" > /dev/null
+  echo $?
+}
+
+install_travel_sample() {
+  URL="http://$1:$REST_PORT/sampleBuckets/install"
+  RESPONSE=$(curl --silent -o /dev/null -w "%{http_code}" "$URL" -X 'POST' -u "$USERNAME:$PASSWORD" --data '["travel-sample"]')
+  echo "install_travel_sample > $RESPONSE"
+}
+
+get_buckets() {
+  URL="http://$1:$REST_PORT/pools/default/buckets"
+  RESPONSE=$(curl --silent --max-time 5 "$URL" -X 'GET' -u "$USERNAME:$PASSWORD")
+  echo "$RESPONSE"
+}
+
+get_rebalance_logs() {
+  URL="http://$1:$REST_PORT/logs/rebalanceReport"
+  RESPONSE=$(curl --silent --max-time 5 "$URL" -X 'GET' -u "$USERNAME:$PASSWORD")
+  echo "$RESPONSE"
+}
+
+node_init() {
+  URL="http://$1:$REST_PORT/nodeInit"
+  RESPONSE=$(curl --silent -o /dev/null -w "%{http_code}" "$URL" -X 'POST' --data "hostname=$1&dataPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&indexPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&eventingPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&javaHome=&analyticsPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata")
+  echo "node_init > $RESPONSE"
+}
+
+create_cluster() {
+  URL="http://$1:$REST_PORT/clusterInit"
+  RESPONSE=$(curl --silent -o /dev/null -w "%{http_code}" "$URL" -X 'POST' --data "hostname=$1&dataPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&indexPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&eventingPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&sendStats=false&services=kv&analyticsPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&javaHome=&clusterName=home&memoryQuota=$MEMORY_AS_MB&afamily=ipv4&afamilyOnly=false&nodeEncryption=off&username=$USERNAME&password=$PASSWORD&port=SAME")
+  echo "create_cluster > $RESPONSE"
+}
+
+join_cluster() {
+  URL="http://$2:$REST_PORT/node/controller/doJoinCluster"
+  RESPONSE=$(curl --silent -o /dev/null -w "%{http_code}" "$URL" -X 'POST' --data "hostname=$1&user=$USERNAME&password=$PASSWORD&newNodeHostname=$2&services=kv")
+  echo "join_cluster > $RESPONSE"
+}
+
+rebalance() {
+  URL="http://$1:$REST_PORT/controller/rebalance"
+  RESPONSE=$(curl --silent -o /dev/null -w "%{http_code}" "$URL" -X 'POST' -u "$USERNAME:$PASSWORD" --data "knownNodes=$2&ejectedNodes=")
+  echo "rebalance > $RESPONSE"
+}
+
+echo "creating nodes"
+for (( i=1; i<=TOTAL_NODE; i++ ))
+do
+  NAME="$NAME_PREFIX$i"
+  docker run -d --pull missing --name $NAME --net couchbase "couchbase:$VERSION"
+done
+
+echo "waiting nodes"
+for (( i=1; i<=TOTAL_NODE; i++ ))
+do
+  NAME="$NAME_PREFIX$i"
+  IP=$(docker inspect $NAME -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
+  KNOWN_NODES="${KNOWN_NODES}ns_1%40${IP}"
+  if [ "$i" != $TOTAL_NODE ]; then
+    KNOWN_NODES="${KNOWN_NODES}%2C"
+  fi
+
+  echo "$NAME has IP=$IP"
+
+  until [[ $(check_db "$IP") = 0 ]]; do
+    >&2 echo "waiting $NAME to be available"
+    sleep 5
+  done
+
+  echo "$NAME ready"
+
+  if [ "$i" == 1 ]; then
+    echo "creating cluster"
+    create_cluster "$IP"
+    MASTER="$IP"
+  else
+    echo "$NAME joining cluster"
+    node_init "$IP"
+    join_cluster "$MASTER" "$IP"
+  fi
+done
+
+echo "rebalancing"
+
+rebalance "$MASTER" "$KNOWN_NODES"
+
+until [[ $(get_rebalance_logs "$MASTER") == *"Rebalance completed"* ]]; do
+  >&2 echo "waiting rebalance to be done"
+  sleep 5
+done
+
+echo "rebalance done"
+
+echo "installing travel-sample"
+
+install_travel_sample "$MASTER"
+
+until [[ $(get_buckets "$MASTER") == *"\"itemCount\":63288"* ]]; do
+  >&2 echo "waiting travel-sample to be installed"
+  sleep 10
+done
+
+echo "travel-sample installed"
+
+URL="http://$MASTER:$REST_PORT"
+
+echo "---------------------------"
+echo "username $USERNAME"
+echo "password $PASSWORD"
+echo "---------------------------"
+echo "ui       $URL"
+echo "---------------------------"
+HOSTS="hosts:"
+HOSTS_AS_GO_ARR="[]string{"
+for (( i=1; i<=TOTAL_NODE; i++ ))
+do
+  NAME="$NAME_PREFIX$i"
+  IP=$(docker inspect $NAME -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
+  HOSTS="$HOSTS\n  - $IP:$REST_PORT"
+  HOSTS_AS_GO_ARR="${HOSTS_AS_GO_ARR}\"$IP:$REST_PORT\""
+  if [ "$i" != $TOTAL_NODE ]; then
+    HOSTS_AS_GO_ARR="${HOSTS_AS_GO_ARR}, "
+  fi
+done
+HOSTS_AS_GO_ARR="${HOSTS_AS_GO_ARR}}"
+echo "$HOSTS"
+echo "---------------------------"
+echo "$HOSTS_AS_GO_ARR"
+echo "---------------------------"

scripts/delete_cluster.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "deleting nodes"
+
+NODES=$(docker network inspect couchbase --format '{{range .Containers}}{{printf "%s\n" .Name}}{{end}}')
+
+echo "$NODES" | while read -r NAME ; do
+   docker rm -f "$NAME"
+done
+
+echo "deleting network"
+
+docker network rm couchbase

test/integration/config.yml

@@ -1,12 +1,19 @@
 hosts:
-  - localhost:8091
+  - 13.37.11.2:8091
+  - 13.37.11.3:8091
+  - 13.37.11.4:8091
+  - 13.37.11.5:8091
+  - 13.37.11.6:8091
+  - 13.37.11.7:8091
+  - 13.37.11.8:8091
+  - 13.37.11.9:8091
+  - 13.37.11.10:8091
+  - 13.37.11.11:8091
+  - 13.37.11.12:8091
+  - 13.37.11.13:8091
 username: user
 password: 123456
-bucketName: dcp-test
-rollbackMitigation:
-  disabled: true
-checkpoint:
-  type: manual
+bucketName: travel-sample
 dcp:
   group:
     name: groupName
@@ -16,4 +23,4 @@ metadata:
   type: couchbase
   readOnly: true
   config:
-    bucket: dcp-test
+    bucket: travel-sample