[meta] upgrade test (elastic#907)

This PR is adding a new upgrade test for every chart.

This test is deploying the oldest **release** version of charts which can be upgraded to latest version, then upgrading using unreleased charts in the repo.

The goal is to detect when some PR is introducing a breaking change.

- Note 1: An upgrade test is already existing for Elasticsearch but had been removed from master branch because upgrade from 7.x to Elasticsearch 8.0.0-SNAPSHOT used by master branch isn't working. The new test allows overriding the Docker image version used during upgrade so we can still test upgrade on master but use the latest 7.x Docker image instead of 8.0.0-SNAPSHOT.

* Note 2:  Metricbeat chart introduced a breaking change in elastic#516 and can be upgraded from any previous version currently, so Metricbeat test is commented and should be uncommented after 7.10.0 release so we can check if new PR are introducing new breaking changes.

* Note 3: Currently charts upgrade are only tested with default values and so cover a small part of charts codebase, we should update it to test optional resources (ie: enable service or ingress, ...).

Fix elastic#396

apm-server/examples/upgrade/Makefile

@@ -0,0 +1,15 @@
+default: test
+
+include ../../../helpers/examples.mk
+
+CHART := apm-server
+RELEASE := helm-apm-server-upgrade
+FROM := 7.6.0	# 7.6.0 is the first release for apm-server
+
+install:
+	../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM)
+
+test: install goss
+
+purge:
+	helm del $(RELEASE)

apm-server/examples/upgrade/README.md

@@ -0,0 +1,21 @@
+# Upgrade
+
+This example will deploy APM Server chart using an old chart version,
+then upgrade it.
+
+
+## Usage
+
+* Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co`
+
+* Deploy [Elasticsearch Helm chart][]: `helm install elasticsearch elastic/elasticsearch`
+
+* Deploy and upgrade APM Server chart with the default values: `make install`
+
+
+## Testing
+
+You can also run [goss integration tests][] using `make test`.
+
+
+[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/apm-server/examples/upgrade/test/goss.yaml

apm-server/examples/upgrade/test/goss.yaml

@@ -0,0 +1,6 @@
+http:
+  http://localhost:8200?pretty:
+    status: 200
+    timeout: 2000
+    body:
+      - "8.0.0"

apm-server/examples/upgrade/values.yaml

@@ -0,0 +1,12 @@
+---
+apmConfig:
+  apm-server.yml: |
+    apm-server:
+      host: "0.0.0.0:8200"
+
+    queue: {}
+    output.file:
+      enabled: false
+
+    output.elasticsearch:
+      hosts: ["http://upgrade-master:9200"]

elasticsearch/examples/upgrade/Makefile

@@ -2,10 +2,12 @@ default: test
 
 include ../../../helpers/examples.mk
 
+CHART := elasticsearch
 RELEASE := helm-es-upgrade
+FROM := 7.4.0	# versions before 7.4.O aren't compatible with Kubernetes >= 1.16.0
 
 install:
-	./scripts/upgrade.sh --release $(RELEASE)
+	../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM)
 
 test: install goss
 

elasticsearch/examples/upgrade/README.md

@@ -1,27 +1,17 @@
 # Upgrade
 
-This example will deploy a 3 node Elasticsearch cluster using an old chart version,
-then upgrade it to version 7.10.0-SNAPSHOT.
-
-The following upgrades are tested:
-- Upgrade from [7.0.0-alpha1][] version on K8S <1.16
-- Upgrade from [7.4.0][] version on K8S >=1.16 (Elasticsearch chart < 7.4.0 are
-not compatible with K8S >= 1.16)
+This example will deploy a 3 node Elasticsearch cluster chart using an old chart
+version, then upgrade it.
 
 
 ## Usage
 
-Running `make install` command will do first install and 7.10.0-SNAPSHOT upgrade.
-
-Note: [jq][] is a requirement for this make target.
+* Deploy and upgrade Elasticsearch chart with the default values: `make install`
 
 
 ## Testing
 
 You can also run [goss integration tests][] using `make test`.
 
 
-[7.0.0-alpha1]: https://github.com/elastic/helm-charts/releases/tag/7.0.0-alpha1
-[7.4.0]: https://github.com/elastic/helm-charts/releases/tag/7.4.0
-[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.10/elasticsearch/examples/upgrade/test/goss.yaml
-[jq]: https://stedolan.github.io/jq/
+[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/upgrade/test/goss.yaml

elasticsearch/examples/upgrade/test/goss.yaml

@@ -3,15 +3,15 @@ http:
     status: 200
     timeout: 2000
     body:
-      - 'green'
+      - "green"
       - '"number_of_nodes":3'
       - '"number_of_data_nodes":3'
 
   http://localhost:9200:
     status: 200
     timeout: 2000
     body:
-      - '"number" : "7.10.0-SNAPSHOT"'
+      - '"number" : "7.10.0"'
       - '"cluster_name" : "upgrade"'
       - '"name" : "upgrade-master-0"'
-      - 'You Know, for Search'
+      - "You Know, for Search"

elasticsearch/examples/upgrade/values.yaml

@@ -0,0 +1,2 @@
+---
+clusterName: upgrade

filebeat/examples/upgrade/Makefile

@@ -0,0 +1,15 @@
+default: test
+
+include ../../../helpers/examples.mk
+
+CHART := filebeat
+RELEASE := helm-filebeat-upgrade
+FROM := 7.9.0 # registry file version 1 not supported error with previous version
+
+install:
+	../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM)
+
+test: install goss
+
+purge:
+	helm del $(RELEASE)

filebeat/examples/upgrade/README.md

@@ -0,0 +1,21 @@
+# Upgrade
+
+This example will deploy Filebeat chart using an old chart version,
+then upgrade it.
+
+
+## Usage
+
+* Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co`
+
+* Deploy [Elasticsearch Helm chart][]: `helm install elasticsearch elastic/elasticsearch`
+
+* Deploy and upgrade Filebeat chart with the default values: `make install`
+
+
+## Testing
+
+You can also run [goss integration tests][] using `make test`.
+
+
+[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/filebeat/examples/upgrade/test/goss.yaml

filebeat/examples/upgrade/test/goss.yaml

@@ -0,0 +1,45 @@
+port:
+  tcp:5066:
+    listening: true
+    ip:
+      - "127.0.0.1"
+
+mount:
+  /usr/share/filebeat/data:
+    exists: true
+  /run/docker.sock:
+    exists: true
+  /var/lib/docker/containers:
+    exists: true
+    opts:
+      - ro
+  /usr/share/filebeat/filebeat.yml:
+    exists: true
+    opts:
+      - ro
+
+user:
+  filebeat:
+    exists: true
+    uid: 1000
+    gid: 1000
+
+http:
+  http://upgrade-master:9200/_cat/indices:
+    status: 200
+    timeout: 2000
+    body:
+      - "filebeat-8.0.0"
+
+file:
+  /usr/share/filebeat/filebeat.yml:
+    exists: true
+    contains:
+      - "add_kubernetes_metadata"
+      - "output.elasticsearch"
+
+command:
+  cd /usr/share/filebeat && filebeat test output:
+    exit-status: 0
+    stdout:
+      - "elasticsearch: http://upgrade-master:9200"

filebeat/examples/upgrade/values.yaml

@@ -0,0 +1,4 @@
+---
+extraEnvs:
+  - name: ELASTICSEARCH_HOSTS
+    value: upgrade-master:9200

helpers/matrix.yml

@@ -16,24 +16,29 @@ KIBANA_SUITE:
   - default
   - oss
   - security
+  - upgrade
 FILEBEAT_SUITE:
   - default
   - oss
   - security
+  - upgrade
 METRICBEAT_SUITE:
   - default
   - oss
   - security
+  #- upgrade TODO: uncomment after 7.10.0 release
 LOGSTASH_SUITE:
   - default
   - oss
   - elasticsearch
   - security
+  - upgrade
 APM_SERVER_SUITE:
   - default
   - oss
   - security
+  - upgrade
 KUBERNETES_VERSION:
-  - '1.15'
-  - '1.16'
-  - '1.17'
+  - "1.15"
+  - "1.16"
+  - "1.17"

helpers/upgrade.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+#
+# upgrade.sh deploy some Helm chart to a specific released version,
+# then upgrade it.
+#
+# An optional version can be specified for Docker image tag to use for upgrade.
+# This is required for master branch because upgrade from Elasticsearch 7.X
+# to 8.0.0-SNAPSHOT doesn't work.
+#
+set -euo pipefail
+
+TO=""
+
+usage() {
+  cat <<-EOF
+	USAGE:
+	  $0 --chart <chart-name> --release <release-name> --from <version> [--to <docker-image-version>]
+	  $0 --help
+
+	OPTIONS:
+    --chart <chart-name>
+      Name of the Elastic Helm chart to install (ie: elasticsearch)
+	  --release <release-name>
+	    Name of the Helm release to install (ie: helm-upgrade-elasticsearch)
+	  --from <version>
+	    Version to use for first install (ie: 7.7.0)
+    --to <docker-image-version>
+      Version of the Docker images to use for upgrade (ie: 7.10.0)
+	EOF
+  exit 1
+}
+
+while [[ $# -gt 0 ]]
+do
+  key="$1"
+
+  case $key in
+    --help)
+      usage
+    ;;
+    --chart)
+      CHART="$2"
+      shift 2
+    ;;
+    --release)
+      RELEASE="$2"
+      shift 2
+    ;;
+    --from)
+      FROM="$2"
+      shift 2
+    ;;
+    --to)
+      TO="--set imageTag=$2"
+      shift 2
+    ;;
+    *)
+      log "Unrecognized argument: '$key'"
+      usage
+    ;;
+  esac
+done
+
+helm repo add elastic https://helm.elastic.co
+
+# Initial install
+printf "Installing %s %s\n" "$RELEASE" "$FROM"
+helm upgrade --wait --timeout=1200s --install --version "$FROM" --values values.yaml "$RELEASE" elastic/"$CHART"
+
+# Upgrade
+printf "Upgrading %s\n" "$RELEASE"
+# shellcheck disable=SC2086
+helm upgrade --wait --timeout=1200s --install --set terminationGracePeriod=121 $TO --values values.yaml "$RELEASE" ../../