Revert "Skip x-pack libbeat tests again as flaky (elastic#10068)" (el…

…astic#10179) (elastic#10260)

* Enable back CM integration suite

This reverts commit 0724101.

And do the following:

- Move all docker-compose.yml file to the version 2.3 format to have
support for `start_period`
- The health check check the cluster health instead of checking that the
host respond.
- Use the `ELASTIC_PASSWORD` variable instead of invoking the CLI.
- DUMP the last health check information and the docker-compose logs

.travis.yml

@@ -12,7 +12,7 @@ env:
   global:
     # Cross-compile for amd64 only to speed up testing.
     - GOX_FLAGS="-arch amd64"
-    - DOCKER_COMPOSE_VERSION=1.11.1
+    - DOCKER_COMPOSE_VERSION=1.21.0
     - GO_VERSION="$(cat .go-version)"
     # Newer versions of minikube fail on travis, see: https://github.com/kubernetes/minikube/issues/2704
     - TRAVIS_MINIKUBE_VERSION=v0.25.2

auditbeat/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '2.3'
 services:
   beat:
     build: ${PWD}/.

filebeat/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '2.3'
 services:
   beat:
     build: ${PWD}/.

heartbeat/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '2.3'
 services:
   beat:
     build: ${PWD}/.

libbeat/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '2.3'
 services:
   beat:
     build: ${PWD}/.

libbeat/scripts/Makefile

@@ -197,7 +197,7 @@ integration-tests: prepare-tests
 .PHONY: integration-tests-environment
 integration-tests-environment:  ## @testing Runs the integration inside a virtual environment. This can be run on any docker-machine (local, remote)
 integration-tests-environment: prepare-tests build-image
-	${DOCKER_COMPOSE} run beat make integration-tests RACE_DETECTOR=$(RACE_DETECTOR) DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME}
+	${DOCKER_COMPOSE} run beat make integration-tests RACE_DETECTOR=$(RACE_DETECTOR) DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} || ${DOCKER_COMPOSE} logs --tail 200
 
 # Runs the system tests
 .PHONY: system-tests

testing/environments/5.x.yml

@@ -1,6 +1,6 @@
 # This is the latest stable 5x release environment.
 
-version: '2.1'
+version: '2.3'
 services:
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:5.6.9

testing/environments/6.0.yml

@@ -1,6 +1,6 @@
 # This is the latest 6.0 environment.
 
-version: '2.1'
+version: '2.3'
 services:
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch-platinum:6.0.1

testing/environments/latest.yml

@@ -1,6 +1,6 @@
 # This is the latest released environment.
 
-version: '2.1'
+version: '2.3'
 services:
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:6.4.0

testing/environments/local.yml

@@ -2,7 +2,7 @@
 # This is useful for testing locally with a full elastic stack setup.
 # All services can be reached through localhost like localhost:5601 for Kibana
 # This is not used for CI as otherwise ports conflicts could happen.
-version: '2.1'
+version: '2.3'
 services:
   kibana:
     ports:

testing/environments/snapshot-oss.yml

@@ -1,6 +1,6 @@
 # This should start the environment with the latest snapshots.
 
-version: '2.1'
+version: '2.3'
 services:
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0-alpha1-SNAPSHOT

testing/environments/snapshot.yml

@@ -1,6 +1,6 @@
 # This should start the environment with the latest snapshots.
 
-version: '2.1'
+version: '2.3'
 services:
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0-SNAPSHOT

x-pack/auditbeat/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '2.3'
 services:
   beat:
     build: ../../auditbeat

x-pack/filebeat/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '2.3'
 services:
   beat:
     build: ../../filebeat

x-pack/functionbeat/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '2.3'
 services:
   beat:
     build: ${PWD}/.

x-pack/libbeat/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '2.3'
 services:
   beat:
     build: ${PWD}/.
@@ -20,33 +20,36 @@ services:
     depends_on:
       elasticsearch: { condition: service_healthy }
       kibana: { condition: service_healthy }
-    healthcheck:
-      interval: 1s
-      retries: 2400
 
   elasticsearch:
     extends:
       file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml
       service: elasticsearch
     healthcheck:
-      test: ["CMD", "curl", "-u", "elastic:changeme", "-f", "http://localhost:9200"]
+      test: ["CMD-SHELL", 'python -c ''import urllib, json; response = urllib.urlopen("http://myelastic:changeme@localhost:9200/_cluster/health"); data = json.loads(response.read()); exit(1) if data["status"] != "green" else exit(0);''']
       retries: 1200
-      interval: 1s
+      interval: 5s
+      start_period: 60s
     environment:
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
       - "network.host="
       - "transport.host=127.0.0.1"
       - "http.host=0.0.0.0"
       - "xpack.security.enabled=true"
       - "xpack.license.self_generated.type=trial"
-    command: bash -c "bin/elasticsearch-keystore create && echo changeme | bin/elasticsearch-keystore add --stdin bootstrap.password && /usr/local/bin/docker-entrypoint.sh eswrapper"
+      - "xpack.security.authc.realms.file1.type=file"
+      - "xpack.security.authc.realms.file1.order=0"
+    command: bash -c "bin/elasticsearch-users useradd myelastic -r superuser -p changeme | /usr/local/bin/docker-entrypoint.sh eswrapper"
 
   kibana:
+    depends_on:
+      - elasticsearch
     extends:
       file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml
       service: kibana
     healthcheck:
-      test: ["CMD-SHELL", 'python -c ''import urllib, json; response = urllib.urlopen("http://elastic:changeme@localhost:5601/api/status"); data = json.loads(response.read()); exit(1) if data["status"]["overall"]["state"] != "green" else exit(0);''']
+      test: ["CMD-SHELL", 'python -c ''import urllib, json; response = urllib.urlopen("http://myelastic:changeme@localhost:5601/api/status"); data = json.loads(response.read()); exit(1) if data["status"]["overall"]["state"] != "green" else exit(0);''']
       retries: 1200
-      interval: 1s
-    command: /usr/local/bin/kibana-docker --xpack.security.enabled=true --elasticsearch.username=elastic --elasticsearch.password=changeme
+      interval: 5s
+      start_period: 60s
+    command: /usr/local/bin/kibana-docker --xpack.security.enabled=true --elasticsearch.username=myelastic --elasticsearch.password=changeme