fix docker compose tests (cadence-workflow#5479)

* wait for cassandra healthy state on ci and local docker compose * remove unused containers on lint
Shaddoll · Dec 16, 2023 · 40018f1 · 40018f1
1 parent 6802914
commit 40018f1
Show file tree

Hide file tree

Showing 5 changed files with 107 additions and 37 deletions.
diff --git a/.buildkite/pipeline-master.yml b/.buildkite/pipeline-master.yml
@@ -30,7 +30,7 @@ steps:
         limit: 1
     plugins:
       - docker-compose#v3.0.0:
-          run: unit-test
+          run: coverage-report
           config: docker/buildkite/docker-compose.yml
 
   - label: ":golang: integration test with cassandra"

diff --git a/.buildkite/pipeline-pull-request.yml b/.buildkite/pipeline-pull-request.yml
@@ -9,7 +9,7 @@ steps:
       queue: "workers"
       docker: "*"
     commands:
-      - "sleep 30"
+      - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code
       - "CASSANDRA_HOST=cassandra make install-schema && make cover_profile" # make install-schema is needed for a server startup test. See main_test.go
     artifact_paths:
       - ".build/coverage/*.out"
@@ -21,7 +21,7 @@ steps:
           run: unit-test
           config: docker/buildkite/docker-compose.yml
 
-  - label: ":golangci-lint: validate code is clean"
+  - label: ":lint: validate code is clean"
     agents:
       queue: "workers"
       docker: "*"
@@ -32,14 +32,16 @@ steps:
         limit: 1
     plugins:
       - docker-compose#v3.0.0:
-          run: unit-test
+          run: coverage-report
           config: docker/buildkite/docker-compose.yml
 
   - label: ":golang: integration test with cassandra"
     agents:
       queue: "workers"
       docker: "*"
-    command: "make cover_integration_profile"
+    commands:
+      - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code
+      - "make cover_integration_profile"
     artifact_paths:
       - ".build/coverage/*.out"
     retry:
@@ -54,7 +56,9 @@ steps:
     agents:
       queue: "workers"
       docker: "*"
-    command: "make cover_integration_profile"
+    command:
+      - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code
+      - "make cover_integration_profile"
     artifact_paths:
       - ".build/coverage/*.out"
     retry:
@@ -68,7 +72,9 @@ steps:
     agents:
       queue: "workers"
       docker: "*"
-    command: "make cover_integration_profile"
+    commands:
+      - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code
+      - "make cover_integration_profile"
     artifact_paths:
       - ".build/coverage/*.out"
     retry:
@@ -82,7 +88,9 @@ steps:
     agents:
       queue: "workers"
       docker: "*"
-    command: "make cover_ndc_profile"
+    commands:
+      - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code
+      - "make cover_ndc_profile"
     artifact_paths:
       - ".build/coverage/*.out"
     retry:
@@ -97,7 +105,9 @@ steps:
     agents:
       queue: "workers"
       docker: "*"
-    command: "make cover_integration_profile"
+    commands:
+      - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code
+      - "make cover_integration_profile"
     artifact_paths:
       - ".build/coverage/*.out"
     retry:
@@ -112,7 +122,9 @@ steps:
     agents:
       queue: "workers"
       docker: "*"
-    command: "make cover_ndc_profile"
+    commands:
+      - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code
+      - "make cover_ndc_profile"
     artifact_paths:
       - ".build/coverage/*.out"
     retry:
@@ -127,7 +139,9 @@ steps:
     agents:
       queue: "workers"
       docker: "*"
-    command: "make cover_integration_profile"
+    commands:
+      - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code
+      - "make cover_integration_profile"
     artifact_paths:
       - ".build/coverage/*.out"
     retry:
@@ -142,7 +156,9 @@ steps:
     agents:
       queue: "workers"
       docker: "*"
-    command: "make cover_ndc_profile"
+    commands:
+      - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code
+      - "make cover_ndc_profile"
     artifact_paths:
       - ".build/coverage/*.out"
     retry:

diff --git a/Makefile b/Makefile
@@ -548,7 +548,7 @@ test_e2e_xdc: bins
 		go test $(TEST_ARG) -coverprofile=$@ "$$dir" $(TEST_TAG) | tee -a test.log; \
 	done;
 
-cover_profile: bins
+cover_profile:
 	$Q mkdir -p $(BUILD)
 	$Q mkdir -p $(COVER_ROOT)
 	$Q echo "mode: atomic" > $(UNIT_COVER_FILE)
@@ -599,12 +599,14 @@ cover_ci: $(COVER_ROOT)/cover.out $(BIN)/goveralls
 	$(BIN)/goveralls -coverprofile=$(COVER_ROOT)/cover.out -service=buildkite || echo Coveralls failed;
 
 install-schema: cadence-cassandra-tool
+	$Q echo installing schema
 	./cadence-cassandra-tool create -k cadence --rf 1
 	./cadence-cassandra-tool -k cadence setup-schema -v 0.0
 	./cadence-cassandra-tool -k cadence update-schema -d ./schema/cassandra/cadence/versioned
 	./cadence-cassandra-tool create -k cadence_visibility --rf 1
 	./cadence-cassandra-tool -k cadence_visibility setup-schema -v 0.0
 	./cadence-cassandra-tool -k cadence_visibility update-schema -d ./schema/cassandra/visibility/versioned
+	$Q echo installed schema
 
 install-schema-mysql: cadence-sql-tool
 	./cadence-sql-tool --user root --pw cadence create --db cadence

diff --git a/docker/buildkite/docker-compose-local.yml b/docker/buildkite/docker-compose-local.yml
@@ -9,6 +9,11 @@ services:
       services-network:
         aliases:
           - cassandra
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
 
   mysql:
     image: mysql:8.0
@@ -86,8 +91,9 @@ services:
     build:
       context: ../../
       dockerfile: ./docker/buildkite/Dockerfile
-    command: make cover_profile
+    command: sh -c "make .just-build && make install-schema && make cover_profile"
     environment:
+      - "CASSANDRA_HOST=cassandra"
       - "CASSANDRA=1"
       - "MYSQL=1"
       - "POSTGRES=1"
@@ -98,12 +104,18 @@ services:
       - "POSTGRES_USER=cadence"
       - "POSTGRES_PASSWORD=cadence"
     depends_on:
-      - cassandra
-      - mysql
-      - postgres
-      - mongo
+      cassandra:
+        condition: service_healthy
+      mysql:
+        condition: service_started
+      postgres:
+        condition: service_started
+      mongo:
+        condition: service_started
     volumes:
       - ../../:/cadence
+      - /cadence/.build/ # ensure we don't mount the build directory
+      - /cadence/.bin/ # ensure we don't mount the bin directory
     networks:
       services-network:
         aliases:
@@ -115,17 +127,23 @@ services:
       dockerfile: ./docker/buildkite/Dockerfile
     command: make cover_integration_profile
     environment:
+      - "CASSANDRA_HOST=cassandra"
       - "CASSANDRA=1"
       - "CASSANDRA_SEEDS=cassandra"
       - "ES_SEEDS=elasticsearch"
       - "KAFKA_SEEDS=kafka"
       - "TEST_TAG=esintegration"
     depends_on:
-      - cassandra
-      - elasticsearch
-      - kafka
+      cassandra:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_started
+      kafka:
+        condition: service_started
     volumes:
       - ../../:/cadence
+      - /cadence/.build/ # ensure we don't mount the build directory
+      - /cadence/.bin/ # ensure we don't mount the bin directory
     networks:
       services-network:
         aliases:
@@ -149,6 +167,8 @@ services:
       - kafka
     volumes:
       - ../../:/cadence
+      - /cadence/.build/ # ensure we don't mount the build directory
+      - /cadence/.bin/ # ensure we don't mount the bin directory
     networks:
       services-network:
         aliases:
@@ -173,6 +193,8 @@ services:
       - kafka
     volumes:
       - ../../:/cadence
+      - /cadence/.build/ # ensure we don't mount the build directory
+      - /cadence/.bin/ # ensure we don't mount the bin directory
     networks:
       services-network:
         aliases:
@@ -184,17 +206,23 @@ services:
       dockerfile: ./docker/buildkite/Dockerfile
     command: make cover_integration_profile EVENTSV2=true
     environment:
+      - "CASSANDRA_HOST=cassandra"
       - "CASSANDRA=1"
       - "CASSANDRA_SEEDS=cassandra"
       - "ES_SEEDS=elasticsearch"
       - "KAFKA_SEEDS=kafka"
       - "TEST_TAG=esintegration"
     depends_on:
-      - cassandra
-      - elasticsearch
-      - kafka
+      cassandra:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_started
+      kafka:
+        condition: service_started
     volumes:
       - ../../:/cadence
+      - /cadence/.build/ # ensure we don't mount the build directory
+      - /cadence/.bin/ # ensure we don't mount the bin directory
     networks:
       services-network:
         aliases:
@@ -206,17 +234,23 @@ services:
       dockerfile: ./docker/buildkite/Dockerfile
     command: make cover_ndc_profile
     environment:
+      - "CASSANDRA_HOST=cassandra"
       - "CASSANDRA=1"
       - "CASSANDRA_SEEDS=cassandra"
       - "ES_SEEDS=elasticsearch"
       - "KAFKA_SEEDS=kafka"
       - "TEST_TAG=esintegration"
     depends_on:
-      - cassandra
-      - elasticsearch
-      - kafka
+      cassandra:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_started
+      kafka:
+        condition: service_started
     volumes:
       - ../../:/cadence
+      - /cadence/.build/ # ensure we don't mount the build directory
+      - /cadence/.bin/ # ensure we don't mount the bin directory
     networks:
       services-network:
         aliases:
@@ -240,6 +274,8 @@ services:
       - kafka
     volumes:
       - ../../:/cadence
+      - /cadence/.build/ # ensure we don't mount the build directory
+      - /cadence/.bin/ # ensure we don't mount the bin directory
     networks:
       services-network:
         aliases:

diff --git a/docker/buildkite/docker-compose.yml b/docker/buildkite/docker-compose.yml
@@ -7,6 +7,11 @@ services:
       services-network:
         aliases:
           - cassandra
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
 
   mysql:
     image: mysql:8.0
@@ -73,6 +78,7 @@ services:
       context: ../../
       dockerfile: ./docker/buildkite/Dockerfile
     environment:
+      - "CASSANDRA_HOST=cassandra"
       - "CASSANDRA=1"
       - "MYSQL=1"
       - "POSTGRES=1"
@@ -86,10 +92,14 @@ services:
       - BUILDKITE_BUILD_ID
       - BUILDKITE_BUILD_NUMBER
     depends_on:
-      - cassandra
-      - mysql
-      - postgres
-      - mongo
+      cassandra:
+        condition: service_healthy
+      mysql:
+        condition: service_started
+      postgres:
+        condition: service_started
+      mongo:
+        condition: service_started
     volumes:
       - ../../:/cadence
       - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
@@ -113,9 +123,12 @@ services:
       - BUILDKITE_BUILD_ID
       - BUILDKITE_BUILD_NUMBER
     depends_on:
-      - cassandra
-      - elasticsearch
-      - kafka
+      cassandra:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_started
+      kafka:
+        condition: service_started
     volumes:
       - ../../:/cadence
       - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
@@ -195,9 +208,12 @@ services:
       - BUILDKITE_BUILD_ID
       - BUILDKITE_BUILD_NUMBER
     depends_on:
-      - cassandra
-      - elasticsearch
-      - kafka
+      cassandra:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_started
+      kafka:
+        condition: service_started
     volumes:
       - ../../:/cadence
       - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent