Simplify docker-compose files through docker-compose extend

Make docker-compose command configurable

libbeat/docker-compose.yml

@@ -1,4 +1,4 @@
-libbeat:
+beat:
   build: ..
   links:
     - redis
@@ -10,8 +10,6 @@ libbeat:
     - LS_HOST=logstash
     - LS_TCP_PORT=5044
     - LS_TLS_PORT=5055
-    - ES_USER=beats
-    - ES_PASS=testing
     # Setup work environment
     - LIBBEAT_PATH=/go/src/github.com/elastic/beats/libbeat
     # TODO: Set to true as soon as stable release is out
@@ -35,6 +33,4 @@ logstash:
     - build/test.env
   links:
     - elasticsearch
-  environment:
-    - ES_USER=beats
-    - ES_PASS=testing
+

libbeat/scripts/Makefile

@@ -9,13 +9,14 @@ GOFILES = $(shell find . -type f -name '*.go')
 SHELL=/bin/bash
 ES_HOST?="elasticsearch"
 BEAT_DIR?=github.com/elastic/beats
-BUILD_DIR=build
+BUILD_DIR?=build
 COVERAGE_DIR=${BUILD_DIR}/coverage
 TIMEOUT?= 90
 BEATNAME?=libbeat
 TEST_ENVIRONMENT?=false
 SYSTEM_TESTS?=false
 GOX_OS?=linux darwin windows solaris freebsd netbsd openbsd
+DOCKER_COMPOSE?=docker-compose -f docker-compose.yml
 
 
 ### BUILDING ###
@@ -99,8 +100,8 @@ integration-tests: prepare-tests
 integration-tests-environment:
 	make prepare-tests
 	make build-image
-	NAME=$$(docker-compose run -d ${BEATNAME} make integration-tests | awk 'END{print}') || exit 1; \
-	echo "docker ${BEATNAME} test container: '$$NAME'"; \
+	NAME=$$(${DOCKER_COMPOSE} run -d beat make integration-tests | awk 'END{print}') || exit 1; \
+	echo "docker beat test container: '$$NAME'"; \
 	docker attach $$NAME; CODE=$$?;\
 	mkdir -p ${COVERAGE_DIR}; \
 	docker cp $$NAME:/go/src/${BEAT_DIR}/${BEATNAME}/${COVERAGE_DIR}/integration.cov $(shell pwd)/${COVERAGE_DIR}/; \
@@ -164,22 +165,24 @@ update:
 # Builds the environment to test beat
 .PHONY: build-image
 build-image: write-environment
-	docker-compose build
+	${DOCKER_COMPOSE} build
 
 # Runs the environment so the redis and elasticsearch can also be used for local development
 # To use it for running the test, set ES_HOST and REDIS_HOST environment variable to the ip of your docker-machine.
 .PHONY: start-environment
 start-environment: stop-environment
-	docker-compose up -d redis elasticsearch logstash
+	${DOCKER_COMPOSE} up -d redis elasticsearch logstash
 
 .PHONY: stop-environment
 stop-environment:
-	-docker-compose stop
-	-docker-compose rm -f
+	-${DOCKER_COMPOSE} stop
+	-${DOCKER_COMPOSE} rm -f
 	-docker ps -a  | grep ${BEATNAME} | grep Exited | awk '{print $$1}' | xargs docker rm
 
 .PHONY: write-environment
 write-environment:
-	mkdir -p build
-	echo "ES_HOST=${ES_HOST}" > build/test.env
-	echo "ES_PORT=9200" >> build/test.env
+	mkdir -p ${BUILD_DIR}
+	echo "ES_HOST=${ES_HOST}" > ${BUILD_DIR}/test.env
+	echo "ES_PORT=9200" >> ${BUILD_DIR}/test.env
+	echo "ES_USER=beats" >> ${BUILD_DIR}/test.env
+	echo "ES_PASS=testing" >> ${BUILD_DIR}/test.env

testing/environments/Makefile

@@ -2,12 +2,12 @@ ENV?=latest.yml
 
 start:
 	# This is run every time to make sure the environment is up-to-date
-	docker-compose -f ${ENV} build
-	docker-compose -f ${ENV} run beat bash
+	docker-compose -f base.yml -f ${ENV} build
+	docker-compose -f base.yml -f ${ENV} run beat bash
 
 stop:
-	docker-compose -f ${ENV} stop
-	docker-compose -f ${ENV} rm -f
+	docker-compose -f base.yml -f ${ENV} stop
+	docker-compose -f base.yml -f ${ENV} rm -f
 
 # Be careful using this command, as it will remove all containers and volumes of your docker-machine
 clean:

testing/environments/base.yml

@@ -0,0 +1,35 @@
+# This is the environment with the last major releases
+beat:
+  build: .
+  links:
+    - elasticsearch
+    - logstash
+    - kibana
+  environment:
+    - LS_HOST=logstash
+  container_name: beat
+
+elasticsearch:
+  image: elasticsearch:latest
+  ports:
+    - "9200:9200"
+  container_name: elasticsearch
+
+logstash:
+  image: logstash:latest
+  links:
+    - elasticsearch
+  ports:
+    - "5044:5044"
+    - "5055:5055"
+  container_name: logstash
+
+kibana:
+  image: kibana:latest
+  links:
+    - elasticsearch
+  environment:
+    - ELASTICSEARCH_URL=http://elasticsearch:9200/
+  ports:
+    - "5601:5601"
+  container_name: kibana

testing/environments/es17-ls15-kb41.yml

@@ -1,35 +1,11 @@
 # This is the environment with the last major releases
-beat:
-  build: .
-  links:
-    - elasticsearch
-    - logstash
-    - kibana
-  environment:
-    - LS_HOST=logstash
-  container_name: beat
+# It extends the base.yml environment
 
 elasticsearch:
   image: elasticsearch:1.7
-  ports:
-    - "9200:9200"
-  container_name: elasticsearch
 
 logstash:
   image: logstash:1.5
-  links:
-    - elasticsearch
-  ports:
-    - "5044:5044"
-    - "5055:5055"
-  container_name: logstash
 
 kibana:
   image: kibana:4.1
-  links:
-    - elasticsearch
-  environment:
-    - ELASTICSEARCH_URL=http://elasticsearch:9200/
-  ports:
-    - "5601:5601"
-  container_name: kibana

testing/environments/latest.yml

@@ -1,36 +1,12 @@
-# This is the latest stable release
-beat:
-  build: .
-  links:
-    - elasticsearch
-    - logstash
-    - kibana
-  environment:
-    - LS_HOST=logstash
-  container_name: beat
+# This is the latest stable release environment
+# It inherits from base.yml
 
 elasticsearch:
   image: elasticsearch:latest
-  ports:
-    - "9200:9200"
-  container_name: elasticsearch
 
 logstash:
   image: logstash:latest
-  links:
-    - elasticsearch
-  ports:
-    - "5044:5044"
-    - "5055:5055"
-  container_name: logstash
 
 kibana:
   build: docker/kibana/
   dockerfile: Dockerfile-4.3.1
-  links:
-    - elasticsearch
-  environment:
-    - ELASTICSEARCH_URL=http://elasticsearch:9200/
-  ports:
-    - "5601:5601"
-  container_name: kibana

testing/environments/snapshot.yml

@@ -1,43 +1,19 @@
 # This should test the environment with the latest snapshots
-beat:
-  build: .
-  links:
-    - elasticsearch
-    - logstash
-    - kibana
-  environment:
-    - LS_HOST=logstash
-  container_name: beat
-
+# This is based on base.yml
 elasticsearch:
   build: ./docker/elasticsearch
   dockerfile: Dockerfile-2.2.0
   command: elasticsearch -Des.network.host=0.0.0.0
-  ports:
-    - "9200:9200"
-  container_name: elasticsearch
 
 logstash:
   build: ./docker/logstash
   dockerfile: Dockerfile-2.2.0
   ports:
     - "5044:5044"
     - "5055:5055"
-  links:
-    - elasticsearch
-  container_name: logstash
-  environment:
-    - ES_HOST=elasticsearch
 
 kibana:
   build: ./docker/kibana
   dockerfile: Dockerfile-4.4.0
-  links:
-    - elasticsearch
-  environment:
-    - ELASTICSEARCH_URL=http://elasticsearch:9200/
-  ports:
-    - "5601:5601"
-  container_name: kibana