From 09d4e6b915dfe517aca79fea2bbf9ab802e0a845 Mon Sep 17 00:00:00 2001 From: Taehyun Kim Date: Sat, 4 May 2019 04:26:22 +0900 Subject: [PATCH 001/115] check only if master node string is empty --- elasticsearch/templates/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 993af76d6..a976a373e 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -242,7 +242,7 @@ spec: cleanup () { while true ; do local master="$(http "/_cat/master?h=node")" - if [[ $master == "{{ template "uname" . }}"* && $master != "${NODE_NAME}" ]]; then + if [[ $master && $master != "${NODE_NAME}" ]]; then echo "This node is not master." break fi From 8fcbca9f611348fe936213defc74e6652357991f Mon Sep 17 00:00:00 2001 From: Taehyun Kim Date: Sat, 4 May 2019 04:43:27 +0900 Subject: [PATCH 002/115] refactor services --- elasticsearch/README.md | 3 +++ elasticsearch/templates/service.yaml | 13 ++++++++++--- elasticsearch/values.yaml | 4 ++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 77d298d03..5ea61e09d 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -88,6 +88,9 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.0.1-alpha1 - | `protocol` | The protocol that will be used for the readinessProbe. Change this to `https` if you have `xpack.security.http.ssl.enabled` set | `http` | | `httpPort` | The http port that Kubernetes will use for the healthchecks and the service. If you change this you will also need to set [http.port](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#_settings) in `extraEnvs` | `9200` | | `transportPort` | The transport port that Kubernetes will use for the service. If you change this you will also need to set [transport port configuration](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#_transport_settings) in `extraEnvs` | `9300` | +| `service.type` | Type of elasticsearch service. [Service Types](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) | `ClusterIP` | +| `service.annotations` | Annotations that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` (https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws) | `{}` | + | `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) for the statefulset. By default Kubernetes will wait for the cluster to be green after upgrading each pod. Setting this to `OnDelete` will allow you to manually delete each pod during upgrades | `RollingUpdate` | | `maxUnavailable` | The [maxUnavailable](https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget) value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group | `1` | | `fsGroup` | The Group ID (GID) for [securityContext.fsGroup](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) so that the Elasticsearch user can read from the persistent volume | `1000` | diff --git a/elasticsearch/templates/service.yaml b/elasticsearch/templates/service.yaml index 6bef0f90c..f488c49fb 100644 --- a/elasticsearch/templates/service.yaml +++ b/elasticsearch/templates/service.yaml @@ -3,7 +3,15 @@ kind: Service apiVersion: v1 metadata: name: {{ template "uname" . }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app: "{{ template "uname" . }}" + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} spec: + type: {{ .Values.service.type }} selector: heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} @@ -26,11 +34,10 @@ metadata: release: {{ .Release.Name | quote }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" app: "{{ template "uname" . }}" - annotations: - # Create endpoints also if the related pod isn't ready - service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" spec: clusterIP: None # This is needed for statefulset hostnames like elasticsearch-0 to resolve + # Create endpoints also if the related pod isn't ready + publishNotReadyAddresses: true selector: app: "{{ template "uname" . }}" ports: diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 8a9caf010..55229838b 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -113,6 +113,10 @@ protocol: http httpPort: 9200 transportPort: 9300 +service: + annotations: {} + type: ClusterIP + updateStrategy: RollingUpdate # This is the max unavailable setting for the pod disruption budget From 14aff5abfb18e03e14918fc4940ece9fe41aa45a Mon Sep 17 00:00:00 2001 From: Taehyun Kim Date: Sat, 4 May 2019 05:02:33 +0900 Subject: [PATCH 003/115] update pytest spec --- elasticsearch/tests/elasticsearch_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 5e04b9de1..199b71b77 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -152,6 +152,8 @@ def test_defaults(): # Service s = r['service'][uname] assert s['metadata']['name'] == uname + assert s['metadata']['annotations'] == {} + assert s['spec']['type'] == 'ClusterIP' assert len(s['spec']['ports']) == 2 assert s['spec']['ports'][0] == { 'name': 'http', 'port': 9200, 'protocol': 'TCP'} @@ -161,6 +163,7 @@ def test_defaults(): # Headless Service h = r['service'][uname + '-headless'] assert h['spec']['clusterIP'] == 'None' + assert s['spec']['publishNotReadyAddresses'] == true assert h['spec']['ports'][0]['name'] == 'http' assert h['spec']['ports'][0]['port'] == 9200 assert h['spec']['ports'][1]['name'] == 'transport' From 8ed5081ec9ca9ead19b169cb1235e0ae34a1ed60 Mon Sep 17 00:00:00 2001 From: Taehyun Kim Date: Sat, 4 May 2019 05:03:43 +0900 Subject: [PATCH 004/115] rearrange values --- elasticsearch/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 55229838b..e53b67e9a 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -114,8 +114,8 @@ httpPort: 9200 transportPort: 9300 service: - annotations: {} type: ClusterIP + annotations: {} updateStrategy: RollingUpdate From 5dc0e483cb9761879b89d6e8e056999472efcd4c Mon Sep 17 00:00:00 2001 From: Taehyun Kim Date: Tue, 7 May 2019 23:20:46 +0900 Subject: [PATCH 005/115] fix test failure, update README --- elasticsearch/README.md | 2 +- elasticsearch/tests/elasticsearch_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 5ea61e09d..9a8b1c025 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -89,7 +89,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.0.1-alpha1 - | `httpPort` | The http port that Kubernetes will use for the healthchecks and the service. If you change this you will also need to set [http.port](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#_settings) in `extraEnvs` | `9200` | | `transportPort` | The transport port that Kubernetes will use for the service. If you change this you will also need to set [transport port configuration](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#_transport_settings) in `extraEnvs` | `9300` | | `service.type` | Type of elasticsearch service. [Service Types](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) | `ClusterIP` | -| `service.annotations` | Annotations that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` (https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws) | `{}` | +| `service.annotations` | Annotations that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` [Annotations](https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws) | `{}` | | `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) for the statefulset. By default Kubernetes will wait for the cluster to be green after upgrading each pod. Setting this to `OnDelete` will allow you to manually delete each pod during upgrades | `RollingUpdate` | | `maxUnavailable` | The [maxUnavailable](https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget) value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group | `1` | diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 199b71b77..be21d3b55 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -163,7 +163,7 @@ def test_defaults(): # Headless Service h = r['service'][uname + '-headless'] assert h['spec']['clusterIP'] == 'None' - assert s['spec']['publishNotReadyAddresses'] == true + assert h['spec']['publishNotReadyAddresses'] == true assert h['spec']['ports'][0]['name'] == 'http' assert h['spec']['ports'][0]['port'] == 9200 assert h['spec']['ports'][1]['name'] == 'transport' From da6c3988c20fbe4c2870482dd952d85cb2021549 Mon Sep 17 00:00:00 2001 From: Taehyun Kim Date: Tue, 7 May 2019 23:36:29 +0900 Subject: [PATCH 006/115] set empty string if curl fails --- elasticsearch/templates/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index a976a373e..35a5db4b1 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -241,7 +241,7 @@ spec: cleanup () { while true ; do - local master="$(http "/_cat/master?h=node")" + local master="$(http "/_cat/master?h=node" || echo "")" if [[ $master && $master != "${NODE_NAME}" ]]; then echo "This node is not master." break From dc4820cffbf169849837f90c8c109c84f64a413b Mon Sep 17 00:00:00 2001 From: Taehyun Kim Date: Thu, 16 May 2019 09:14:35 +0900 Subject: [PATCH 007/115] change required master prefix --- elasticsearch/templates/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 35a5db4b1..324bb8d38 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -242,7 +242,7 @@ spec: cleanup () { while true ; do local master="$(http "/_cat/master?h=node" || echo "")" - if [[ $master && $master != "${NODE_NAME}" ]]; then + if [[ $master == "{{ template "masterService" . }}"* && $master != "${NODE_NAME}" ]]; then echo "This node is not master." break fi From 1b6baf9248ceebf8457cab3ffa8dc50b0dd643cc Mon Sep 17 00:00:00 2001 From: Sathya Balakrishnan Date: Wed, 29 May 2019 18:14:04 -0700 Subject: [PATCH 008/115] add capability to specify alternalte scheduler Signed-off-by: Sathya Balakrishnan --- elasticsearch/README.md | 1 + elasticsearch/templates/statefulset.yaml | 3 +++ elasticsearch/values.yaml | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index f8024616e..cd9c43e66 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -100,6 +100,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.1.0 --set im | `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) so that you can target specific nodes for your Elasticsearch cluster | `{}` | | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | | `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Elasticsearch service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | +| `schedulerName` | Name of the alternate scheduler (e.g. stork) | `nil` | ## Try it out diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 00f552f28..cc78da29e 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -45,6 +45,9 @@ spec: configchecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} {{- end }} spec: + {{- if .Values.schedulerName }} + schedulerName: "{{ .Values.schedulerName }}" + {{- end }} securityContext: fsGroup: {{ .Values.fsGroup }} {{- with .Values.tolerations }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index af8c6e1ff..f61a375ca 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -142,6 +142,11 @@ readinessProbe: # https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#request-params wait_for_status clusterHealthCheckParams: "wait_for_status=green&timeout=1s" +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: + imagePullSecrets: [] nodeSelector: {} tolerations: [] From 8803493bd12f6d7af4ac925eb8367c827d84d7e9 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 3 Jun 2019 11:23:47 +0200 Subject: [PATCH 009/115] [elasticsearch] Add instructions for how to enable snapshots Closes #142 --- elasticsearch/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index f8024616e..9ab75ef07 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -190,6 +190,13 @@ There are a couple reasons we recommend this. subPath: elasticsearch.keystore ``` +#### How to enable snapshotting? + +1. Install your [snapshot plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository.html) into a custom docker image following the [how to install plugins guide](/elasticsearch/README.md#how-to-install-plugins) +2. Add any required secrets or credentials into an Elasticsearch keystore following the [how to use the keystore guide](/elasticsearch/README.md#how-to-use-the-keystore) +3. Configure the [snapshot repository](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html) as you normally would. +4. To automate snapshots you can use a tool like [curator](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/snapshot.html). In the future there are plans to have Elasticsearch manage automated snapshots with [Snapshot Lifecycle Management](https://github.com/elastic/elasticsearch/issues/38461). + ### Local development environments From e2c5b6bf91f4106484b0a7444981f169f9cc5c7e Mon Sep 17 00:00:00 2001 From: Sathya Balakrishnan Date: Mon, 3 Jun 2019 09:40:04 -0700 Subject: [PATCH 010/115] update README/values; Add test for schedulerName Signed-off-by: Sathya Balakrishnan --- elasticsearch/README.md | 2 +- elasticsearch/tests/elasticsearch_test.py | 12 ++++++++++++ elasticsearch/values.yaml | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index cd9c43e66..8340961b1 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -100,7 +100,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.1.0 --set im | `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) so that you can target specific nodes for your Elasticsearch cluster | `{}` | | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | | `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Elasticsearch service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | -| `schedulerName` | Name of the alternate scheduler (e.g. stork) | `nil` | +| `schedulerName` | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods) | `nil` | ## Try it out diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 2232a89e2..a6acd1024 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -569,3 +569,15 @@ def test_priority_class_name(): r = helm_template(config) priority_class_name = r['statefulset'][uname]['spec']['template']['spec']['priorityClassName'] assert priority_class_name == "highest" + + +def test_scheduler_name(): + r = helm_template('') + spec = r['statefulset'][uname]['spec']['template']['spec'] + assert 'schedulerName' not in spec + + config = ''' +schedulerName: "stork" +''' + r = helm_template(config) + assert r['statefulset'][uname]['spec']['template']['spec']['schedulerName'] == "stork" diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index f61a375ca..a601a86f7 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -142,7 +142,7 @@ readinessProbe: # https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#request-params wait_for_status clusterHealthCheckParams: "wait_for_status=green&timeout=1s" -## Use an alternate scheduler, e.g. "stork". +## Use an alternate scheduler. ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ ## # schedulerName: From 4d62e81ed85c71ad4384fb496fb26466935b8a4a Mon Sep 17 00:00:00 2001 From: natebwangsut Date: Thu, 6 Jun 2019 15:19:09 +0700 Subject: [PATCH 011/115] Update Kibana service.yaml - Added an option to add annotations(s) to service resource. Signed-off-by: natebwangsut --- kibana/templates/service.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kibana/templates/service.yaml b/kibana/templates/service.yaml index 24adfb7b4..86dda2f3a 100644 --- a/kibana/templates/service.yaml +++ b/kibana/templates/service.yaml @@ -7,6 +7,10 @@ metadata: app: {{ .Chart.Name }} release: {{ .Release.Name | quote }} heritage: {{ .Release.Service }} +{{- with .Values.service.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} spec: type: {{ .Values.service.type }} ports: From 36baa21fdb558d67001740e2fa717764f98c1fc2 Mon Sep 17 00:00:00 2001 From: natebwangsut Date: Thu, 6 Jun 2019 16:20:56 +0700 Subject: [PATCH 012/115] Added README section about k8s service --- kibana/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/kibana/README.md b/kibana/README.md index 0e14afb1a..e0e12f1a7 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -65,6 +65,7 @@ helm install --name kibana elastic/kibana --version 7.1.0 --set imageTag=7.1.0 | `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) so that you can target specific nodes for your Kibana instances | `{}` | | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | | `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Kibana service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | +| `service` | Configurable [service](https://kubernetes.io/docs/concepts/services-networking/service/) to expose the Kibana service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | ## Examples From 856ecf5385dfc4c21a4cdc8649e97e7220f9726e Mon Sep 17 00:00:00 2001 From: natebwangsut Date: Thu, 6 Jun 2019 16:27:52 +0700 Subject: [PATCH 013/115] Added service annotations example --- kibana/values.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kibana/values.yaml b/kibana/values.yaml index 95e576e36..e7bfac250 100755 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -74,6 +74,12 @@ updateStrategy: service: type: ClusterIP port: 5601 + annotations: {} + # cloud.google.com/load-balancer-type: "Internal" + # service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 + # service.beta.kubernetes.io/azure-load-balancer-internal: "true" + # service.beta.kubernetes.io/openstack-internal-load-balancer: "true" + # service.beta.kubernetes.io/cce-load-balancer-internal-vpc: "true" ingress: enabled: false From 9a8fb017a03860d7f80457c50b8b22e3e8b2eb9c Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Thu, 6 Jun 2019 16:27:13 +0200 Subject: [PATCH 014/115] Update issue template to include `helm get release` output --- .github/ISSUE_TEMPLATE/bug_report.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 0f629a421..dc4801407 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -11,10 +11,11 @@ about: Create a report to help us improve **Helm Version:** -**Values.yaml:** +**`helm get release` output** + +e.g. `helm get elasticsearch` (replace `elasticsearch` with the name of your helm release) ``` -key: value ``` **Describe the bug:** From df503ab8111d9104a0ce157461bce93de71eec4c Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Thu, 6 Jun 2019 16:48:23 +0200 Subject: [PATCH 015/115] Release 7.1.1 --- CHANGELOG.md | 14 ++++++++++++++ elasticsearch/Chart.yaml | 4 ++-- elasticsearch/README.md | 10 +++++----- elasticsearch/examples/default/test/goss.yaml | 2 +- elasticsearch/examples/oss/test/goss.yaml | 2 +- elasticsearch/examples/upgrade/test/goss.yaml | 2 +- elasticsearch/values.yaml | 2 +- filebeat/Chart.yaml | 4 ++-- filebeat/README.md | 10 +++++----- filebeat/examples/default/test/goss.yaml | 4 ++-- filebeat/examples/oss/test/goss.yaml | 2 +- filebeat/examples/security/test/goss.yaml | 2 +- filebeat/values.yaml | 2 +- helpers/bumper.py | 2 +- kibana/Chart.yaml | 4 ++-- kibana/README.md | 10 +++++----- kibana/examples/default/test/goss.yaml | 2 +- kibana/values.yaml | 2 +- 18 files changed, 47 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27674b795..96fb00293 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ * [#128](https://github.com/elastic/helm-charts/pull/128) - @Crazybus - Add ci jobs for metricbeat * [#127](https://github.com/elastic/helm-charts/pull/127) - @Crazybus - WIP add metricbeat chart +--- +## 7.1.1 - 2019/06/06 + +* 7.1.1 as the default stack version +* Helm 2.14.0 as the tested version. Helm 2.14.0 has some extra validation built in which caused an issue with an [invalid field in the filebeat chart](https://github.com/elastic/helm-charts/issues/136). + +### Elasticsearch + +* [#146](https://github.com/elastic/helm-charts/pull/146) - @Crazybus - Add instructions for how to enable snapshots + +### Filebeat + +* [#140](https://github.com/elastic/helm-charts/pull/140) - @Crazybus - Remove fsGroup from container level security context + --- ## 7.1.0 - 2019/05/21 diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml index fa959b280..ad42aa16b 100755 --- a/elasticsearch/Chart.yaml +++ b/elasticsearch/Chart.yaml @@ -4,8 +4,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: elasticsearch -version: 7.1.0 -appVersion: 7.1.0 +version: 7.1.1 +appVersion: 7.1.1 sources: - https://github.com/elastic/elasticsearch icon: https://helm.elastic.co/icons/elasticsearch.png diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 9ab75ef07..5fb3fc262 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -33,7 +33,7 @@ If you currently have a cluster deployed with the [helm/charts stable](https://g ``` * Install it ``` - helm install --name elasticsearch elastic/elasticsearch --version 7.1.0 + helm install --name elasticsearch elastic/elasticsearch --version 7.1.1 ``` ## Compatibility @@ -42,14 +42,14 @@ This chart is tested with the latest supported versions. The currently tested ve | 5.x | 6.x | 7.x | | ------ | ----- | ----- | -| 5.6.16 | 6.8.0 | 7.1.0 | +| 5.6.16 | 6.8.0 | 7.1.1 | Examples of installing older major versions can be found in the [examples](./examples) directory. -While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.1.0` of Elasticsearch it would look like this: +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.1.1` of Elasticsearch it would look like this: ``` -helm install --name elasticsearch elastic/elasticsearch --version 7.1.0 --set imageTag=7.1.0 +helm install --name elasticsearch elastic/elasticsearch --version 7.1.1 --set imageTag=7.1.1 ``` @@ -71,7 +71,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.1.0 --set im | `extraInitContainers` | Additional init containers to be passed to the `tpl` function | | | `secretMounts` | Allows you easily mount a secret as a file inside the statefulset. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | | `image` | The Elasticsearch docker image | `docker.elastic.co/elasticsearch/elasticsearch` | -| `imageTag` | The Elasticsearch docker image tag | `7.1.0` | +| `imageTag` | The Elasticsearch docker image tag | `7.1.1` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Elasticsearch pods | `{}` | | `esJavaOpts` | [Java options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for Elasticsearch. This is where you should configure the [jvm heap size](https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html) | `-Xmx1g -Xms1g` | diff --git a/elasticsearch/examples/default/test/goss.yaml b/elasticsearch/examples/default/test/goss.yaml index 29e29dd98..5d1613971 100644 --- a/elasticsearch/examples/default/test/goss.yaml +++ b/elasticsearch/examples/default/test/goss.yaml @@ -21,7 +21,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "7.1.0"' + - '"number" : "7.1.1"' - '"cluster_name" : "elasticsearch"' - '"name" : "elasticsearch-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/examples/oss/test/goss.yaml b/elasticsearch/examples/oss/test/goss.yaml index 305b16a45..1169bd730 100644 --- a/elasticsearch/examples/oss/test/goss.yaml +++ b/elasticsearch/examples/oss/test/goss.yaml @@ -11,7 +11,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "7.1.0"' + - '"number" : "7.1.1"' - '"cluster_name" : "oss"' - '"name" : "oss-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/examples/upgrade/test/goss.yaml b/elasticsearch/examples/upgrade/test/goss.yaml index 95ad14304..e5452a89a 100644 --- a/elasticsearch/examples/upgrade/test/goss.yaml +++ b/elasticsearch/examples/upgrade/test/goss.yaml @@ -11,7 +11,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "7.1.0"' + - '"number" : "7.1.1"' - '"cluster_name" : "upgrade"' - '"name" : "upgrade-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index af8c6e1ff..c1686e5d7 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -43,7 +43,7 @@ secretMounts: [] # path: /usr/share/elasticsearch/config/certs image: "docker.elastic.co/elasticsearch/elasticsearch" -imageTag: "7.1.0" +imageTag: "7.1.1" imagePullPolicy: "IfNotPresent" podAnnotations: {} diff --git a/filebeat/Chart.yaml b/filebeat/Chart.yaml index 7596ab7f8..1137487c6 100755 --- a/filebeat/Chart.yaml +++ b/filebeat/Chart.yaml @@ -4,8 +4,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: filebeat -version: 7.1.0 -appVersion: 7.1.0 +version: 7.1.1 +appVersion: 7.1.1 sources: - https://github.com/elastic/beats icon: https://helm.elastic.co/icons/filebeat.png diff --git a/filebeat/README.md b/filebeat/README.md index d63fa493d..ba271f7da 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -21,7 +21,7 @@ This helm chart is a lightweight way to configure and run our official [Filebeat ``` * Install it ``` - helm install --name filebeat elastic/filebeat --version 7.1.0 + helm install --name filebeat elastic/filebeat --version 7.1.1 ``` ## Compatibility @@ -30,14 +30,14 @@ This chart is tested with the latest supported versions. The currently tested ve | 5.x | 6.x | 7.x | | ------ | ----- | ----- | -| 5.6.16 | 6.8.0 | 7.1.0 | +| 5.6.16 | 6.8.0 | 7.1.1 | Examples of installing older major versions can be found in the [examples](./examples) directory. -While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.1.0` of Filebeat it would look like this: +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.1.1` of Filebeat it would look like this: ``` -helm install --name filebeat elastic/filebeat --version 7.1.0 --set imageTag=7.1.0 +helm install --name filebeat elastic/filebeat --version 7.1.1 --set imageTag=7.1.1 ``` @@ -50,7 +50,7 @@ helm install --name filebeat elastic/filebeat --version 7.1.0 --set imageTag=7.1 | `extraVolumes` | Any extra volumes to define for the pod | `[]` | | `hostPathRoot` | Fully-qualified [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) that will be used to persist Filebeat registry data | `/var/lib` | | `image` | The Filebeat docker image | `docker.elastic.co/beats/filebeat` | -| `imageTag` | The Filebeat docker image tag | `7.1.0` | +| `imageTag` | The Filebeat docker image tag | `7.1.1` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | | `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles. | `true` | diff --git a/filebeat/examples/default/test/goss.yaml b/filebeat/examples/default/test/goss.yaml index dcd9017bb..a06263f7d 100644 --- a/filebeat/examples/default/test/goss.yaml +++ b/filebeat/examples/default/test/goss.yaml @@ -29,7 +29,7 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-7.1.0' + - 'filebeat-7.1.1' file: /usr/share/filebeat/filebeat.yml: @@ -44,4 +44,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://elasticsearch-master:9200' - - 'version: 7.1.0' + - 'version: 7.1.1' diff --git a/filebeat/examples/oss/test/goss.yaml b/filebeat/examples/oss/test/goss.yaml index fc43e5d28..8853288c3 100644 --- a/filebeat/examples/oss/test/goss.yaml +++ b/filebeat/examples/oss/test/goss.yaml @@ -19,4 +19,4 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-7.1.0' + - 'filebeat-7.1.1' diff --git a/filebeat/examples/security/test/goss.yaml b/filebeat/examples/security/test/goss.yaml index b41c847d4..ed8a1c76b 100644 --- a/filebeat/examples/security/test/goss.yaml +++ b/filebeat/examples/security/test/goss.yaml @@ -3,7 +3,7 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-7.1.0' + - 'filebeat-7.1.1' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 76c79aebc..1bb780110 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -34,7 +34,7 @@ extraVolumes: [] hostPathRoot: /var/lib image: "docker.elastic.co/beats/filebeat" -imageTag: "7.1.0" +imageTag: "7.1.1" imagePullPolicy: "IfNotPresent" imagePullSecrets: [] diff --git a/helpers/bumper.py b/helpers/bumper.py index f79341700..d08709f34 100755 --- a/helpers/bumper.py +++ b/helpers/bumper.py @@ -10,7 +10,7 @@ versions = { 5: '5.6.16', 6: '6.8.0', - 7: '7.1.0', + 7: '7.1.1', } file_patterns = [ diff --git a/kibana/Chart.yaml b/kibana/Chart.yaml index ace954179..9c538a3d7 100755 --- a/kibana/Chart.yaml +++ b/kibana/Chart.yaml @@ -4,8 +4,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: kibana -version: 7.1.0 -appVersion: 7.1.0 +version: 7.1.1 +appVersion: 7.1.1 sources: - https://github.com/elastic/kibana icon: https://helm.elastic.co/icons/kibana.png diff --git a/kibana/README.md b/kibana/README.md index 0e14afb1a..fea6006fb 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -17,7 +17,7 @@ This helm chart is a lightweight way to configure and run our official [Kibana d ``` * Install it ``` - helm install --name kibana elastic/kibana --version 7.1.0 + helm install --name kibana elastic/kibana --version 7.1.1 ``` ## Compatibility @@ -26,14 +26,14 @@ This chart is tested with the latest supported versions. The currently tested ve | 5.x | 6.x | 7.x | | ------ | ----- | ----- | -| 5.6.16 | 6.8.0 | 7.1.0 | +| 5.6.16 | 6.8.0 | 7.1.1 | Examples of installing older major versions can be found in the [examples](./examples) directory. -While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.1.0` of Kibana it would look like this: +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.1.1` of Kibana it would look like this: ``` -helm install --name kibana elastic/kibana --version 7.1.0 --set imageTag=7.1.0 +helm install --name kibana elastic/kibana --version 7.1.1 --set imageTag=7.1.1 ``` ## Configuration @@ -46,7 +46,7 @@ helm install --name kibana elastic/kibana --version 7.1.0 --set imageTag=7.1.0 | `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | | `secretMounts` | Allows you easily mount a secret as a file inside the deployment. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | | `image` | The Kibana docker image | `docker.elastic.co/kibana/kibana` | -| `imageTag` | The Kibana docker image tag | `7.1.0` | +| `imageTag` | The Kibana docker image tag | `7.1.1` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 2Gi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | | `protocol` | The protocol that will be used for the readinessProbe. Change this to `https` if you have `server.ssl.enabled: true` set | `http` | diff --git a/kibana/examples/default/test/goss.yaml b/kibana/examples/default/test/goss.yaml index 33561424e..5710496ce 100644 --- a/kibana/examples/default/test/goss.yaml +++ b/kibana/examples/default/test/goss.yaml @@ -3,7 +3,7 @@ http: status: 200 timeout: 2000 body: - - '"number":"7.1.0"' + - '"number":"7.1.1"' http://localhost:5601/app/kibana: status: 200 diff --git a/kibana/values.yaml b/kibana/values.yaml index 95e576e36..297e9df32 100755 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -21,7 +21,7 @@ secretMounts: [] # path: /usr/share/elasticsearch/config/certs image: "docker.elastic.co/kibana/kibana" -imageTag: "7.1.0" +imageTag: "7.1.1" imagePullPolicy: "IfNotPresent" resources: From e6ef1a25b8938d09f570ddb68e0017f2fcaf8e44 Mon Sep 17 00:00:00 2001 From: natebwangsut Date: Thu, 6 Jun 2019 22:50:05 +0700 Subject: [PATCH 016/115] Added kibana_test for service annotation --- kibana/tests/kibana_test.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/kibana/tests/kibana_test.py b/kibana/tests/kibana_test.py index 58f775b82..6ac34bd3f 100644 --- a/kibana/tests/kibana_test.py +++ b/kibana/tests/kibana_test.py @@ -41,6 +41,9 @@ def test_defaults(): assert r['deployment'][name]['spec']['strategy']['type'] == 'Recreate' + # Make sure that the default 'annotation' dictionary is empty + assert not r['service'][name]['metadata']['annotations'] + def test_overriding_the_elasticsearch_hosts(): config = ''' elasticsearchHosts: 'http://hello.world' @@ -252,3 +255,23 @@ def test_priority_class_name(): r = helm_template(config) priority_class_name = r['deployment'][name]['spec']['template']['spec']['priorityClassName'] assert priority_class_name == "highest" + + +def test_service_annotatations(): + config = ''' +service: + annotations: + cloud.google.com/load-balancer-type: "Internal" + ''' + r = helm_template(config) + s = r['service'][name]['metadata']['annotations']['cloud.google.com/load-balancer-type'] + assert s == "Internal" + + config = ''' +service: + annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 + ''' + r = helm_template(config) + s = r['service'][name]['metadata']['annotations']['service.beta.kubernetes.io/aws-load-balancer-internal'] + assert s == "0.0.0.0/0" \ No newline at end of file From 394003c04e2804286163ea2bb32d76674e9a5a92 Mon Sep 17 00:00:00 2001 From: natebwangsut Date: Thu, 6 Jun 2019 22:53:55 +0700 Subject: [PATCH 017/115] Fix wrong README.md service defaults --- kibana/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kibana/README.md b/kibana/README.md index e0e12f1a7..6efebd7e1 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -65,7 +65,7 @@ helm install --name kibana elastic/kibana --version 7.1.0 --set imageTag=7.1.0 | `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) so that you can target specific nodes for your Kibana instances | `{}` | | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | | `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Kibana service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | -| `service` | Configurable [service](https://kubernetes.io/docs/concepts/services-networking/service/) to expose the Kibana service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | +| `service` | Configurable [service](https://kubernetes.io/docs/concepts/services-networking/service/) to expose the Kibana service. See [`values.yaml`](./values.yaml) for an example | `type: ClusterIP`
`port: 5601`
`annotations: {}` | ## Examples From e5d888a1dfe370737719a5271f7a865c7bdb1484 Mon Sep 17 00:00:00 2001 From: Sathya Balakrishnan Date: Thu, 6 Jun 2019 10:47:40 -0700 Subject: [PATCH 018/115] update values for default schedulerName Signed-off-by: Sathya Balakrishnan --- elasticsearch/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index a601a86f7..788769eeb 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -145,7 +145,7 @@ clusterHealthCheckParams: "wait_for_status=green&timeout=1s" ## Use an alternate scheduler. ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ ## -# schedulerName: +schedulerName: "" imagePullSecrets: [] nodeSelector: {} From 613c6a4ead828a8830d2d25ae9c24f5df0539514 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Fri, 7 Jun 2019 08:26:15 +0200 Subject: [PATCH 019/115] Add pull request #151 to changelog for next release --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96fb00293..761242a04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ * [#127](https://github.com/elastic/helm-charts/pull/127) - @Crazybus - WIP add metricbeat chart --- -## 7.1.1 - 2019/06/06 +## 7.1.1 - 2019/06/07 * 7.1.1 as the default stack version * Helm 2.14.0 as the tested version. Helm 2.14.0 has some extra validation built in which caused an issue with an [invalid field in the filebeat chart](https://github.com/elastic/helm-charts/issues/136). @@ -15,6 +15,10 @@ * [#146](https://github.com/elastic/helm-charts/pull/146) - @Crazybus - Add instructions for how to enable snapshots +### Kibana + +* [#151](https://github.com/elastic/helm-charts/pull/151) - @natebwangsut - Added an option to add annotations(s) to service resource + ### Filebeat * [#140](https://github.com/elastic/helm-charts/pull/140) - @Crazybus - Remove fsGroup from container level security context From 920417c10b0159a41596b5e0f66c6a3b5023d7a3 Mon Sep 17 00:00:00 2001 From: natebwangsut Date: Fri, 7 Jun 2019 15:13:48 +0700 Subject: [PATCH 020/115] Fix annotation unit test error --- kibana/tests/kibana_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kibana/tests/kibana_test.py b/kibana/tests/kibana_test.py index 6ac34bd3f..defc23022 100644 --- a/kibana/tests/kibana_test.py +++ b/kibana/tests/kibana_test.py @@ -42,7 +42,7 @@ def test_defaults(): assert r['deployment'][name]['spec']['strategy']['type'] == 'Recreate' # Make sure that the default 'annotation' dictionary is empty - assert not r['service'][name]['metadata']['annotations'] + assert 'annotations' not in r['service'][name]['metadata'] def test_overriding_the_elasticsearch_hosts(): config = ''' From d7bbd0aa155bd6bbad438fc44ba4a5ceef938598 Mon Sep 17 00:00:00 2001 From: Taehyun Kim Date: Fri, 7 Jun 2019 17:42:11 +0900 Subject: [PATCH 021/115] fix README.md --- elasticsearch/README.md | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 7360aa2ed..3444e9440 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -13,6 +13,7 @@ This helm chart is a lightweight way to configure and run our official [Elastics * 1GB of RAM for the JVM heap ## Usage notes and getting started + * This repo includes a number of [example](./examples) configurations which can be used as a reference. They are also used in the automated testing of this chart * Automated testing of this chart is currently only run against GKE (Google Kubernetes Engine). If you are using a different Kubernetes provider you will likely need to adjust the `storageClassName` in the `volumeClaimTemplate` * The default storage class for GKE is `standard` which by default will give you `pd-ssd` type persistent volumes. This is network attached storage and will not perform as well as local storage. If you are using Kubernetes version 1.10 or greater you can use [Local PersistentVolumes](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd) for increased performance @@ -52,7 +53,6 @@ While only the latest releases are tested, it is possible to easily install old helm install --name elasticsearch elastic/elasticsearch --version 7.1.1 --set imageTag=7.1.1 ``` - ## Configuration | Parameter | Description | Default | @@ -89,9 +89,8 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.1.1 --set im | `protocol` | The protocol that will be used for the readinessProbe. Change this to `https` if you have `xpack.security.http.ssl.enabled` set | `http` | | `httpPort` | The http port that Kubernetes will use for the healthchecks and the service. If you change this you will also need to set [http.port](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#_settings) in `extraEnvs` | `9200` | | `transportPort` | The transport port that Kubernetes will use for the service. If you change this you will also need to set [transport port configuration](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#_transport_settings) in `extraEnvs` | `9300` | -| `service.type` | Type of elasticsearch service. [Service Types](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) | `ClusterIP` | -| `service.annotations` | Annotations that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` [Annotations](https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws) | `{}` | - +| `service.type` | Type of elasticsearch service. [Service Types](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) | `ClusterIP` | +| `service.annotations` | Annotations that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` [Annotations](https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws) | `{}` | | `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) for the statefulset. By default Kubernetes will wait for the cluster to be green after upgrading each pod. Setting this to `OnDelete` will allow you to manually delete each pod during upgrades | `RollingUpdate` | | `maxUnavailable` | The [maxUnavailable](https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget) value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group | `1` | | `fsGroup` | The Group ID (GID) for [securityContext.fsGroup](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) so that the Elasticsearch user can read from the persistent volume | `1000` | @@ -130,7 +129,7 @@ make A cluster with X-Pack security enabled -* Generate SSL certificates following the [official docs]( https://www.elastic.co/guide/en/elasticsearch/reference/6.7/configuring-tls.html#node-certificates) +* Generate SSL certificates following the [official docs](https://www.elastic.co/guide/en/elasticsearch/reference/6.7/configuring-tls.html#node-certificates) * Create Kubernetes secrets for authentication credentials and certificates ``` kubectl create secret generic elastic-credentials --from-literal=password=changeme --from-literal=username=elastic @@ -142,6 +141,7 @@ A cluster with X-Pack security enabled make ``` * Attach into one of the containers + ``` kubectl exec -ti $(kubectl get pods -l release=helm-es-security -o name | awk -F'/' '{ print $NF }' | head -n 1) bash ``` @@ -181,17 +181,17 @@ There are a couple reasons we recommend this. #### How to use the keystore? 1. Create a Kubernetes secret containing the [keystore](https://www.elastic.co/guide/en/elasticsearch/reference/current/secure-settings.html) - ``` - $ kubectl create secret generic elasticsearch-keystore --from-file=./elasticsearch.keystore - ``` + ``` + $ kubectl create secret generic elasticsearch-keystore --from-file=./elasticsearch.keystore + ``` 2. Mount it into the container via `secretMounts` - ``` - secretMounts: - - name: elasticsearch-keystore - secretName: elasticsearch-keystore - path: /usr/share/elasticsearch/config/elasticsearch.keystore - subPath: elasticsearch.keystore - ``` + ``` + secretMounts: + - name: elasticsearch-keystore + secretName: elasticsearch-keystore + path: /usr/share/elasticsearch/config/elasticsearch.keystore + subPath: elasticsearch.keystore + ``` #### How to enable snapshotting? @@ -200,7 +200,6 @@ There are a couple reasons we recommend this. 3. Configure the [snapshot repository](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html) as you normally would. 4. To automate snapshots you can use a tool like [curator](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/snapshot.html). In the future there are plans to have Elasticsearch manage automated snapshots with [Snapshot Lifecycle Management](https://github.com/elastic/elasticsearch/issues/38461). - ### Local development environments This chart is designed to run on production scale Kubernetes clusters with multiple nodes, lots of memory and persistent storage. For that reason it can be a bit tricky to run them against local Kubernetes environments such as minikube. Below are some examples of how to get this working locally. @@ -221,7 +220,6 @@ make Note that if `helm` or `kubectl` timeouts occur, you may consider creating a minikube VM with more CPU cores or memory allocated. - #### Docker for Mac - Kubernetes It is also possible to run this chart with the built in Kubernetes cluster that comes with [docker-for-mac](https://docs.docker.com/docker-for-mac/kubernetes/). @@ -269,6 +267,7 @@ make test Integration tests are run using [goss](https://github.com/aelsabbahy/goss/blob/master/docs/manual.md) which is a serverspec like tool written in golang. See [goss.yaml](examples/default/test/goss.yaml) for an example of what the tests look like. To run the goss tests against the default example: + ``` cd examples/default make goss From 1a911be128de35a07f0b4a81a9716b470cc6b63f Mon Sep 17 00:00:00 2001 From: Taehyun Kim Date: Mon, 10 Jun 2019 16:27:58 +0900 Subject: [PATCH 022/115] fix test failure --- elasticsearch/tests/elasticsearch_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 935320e68..ef020cda6 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -163,7 +163,7 @@ def test_defaults(): # Headless Service h = r['service'][uname + '-headless'] assert h['spec']['clusterIP'] == 'None' - assert h['spec']['publishNotReadyAddresses'] == true + assert h['spec']['publishNotReadyAddresses'] == True assert h['spec']['ports'][0]['name'] == 'http' assert h['spec']['ports'][0]['port'] == 9200 assert h['spec']['ports'][1]['name'] == 'transport' From b1f9f34efdca39de9cb719c02e79ea44dacad0df Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 11 Jun 2019 16:42:43 +0200 Subject: [PATCH 023/115] Update beta notice and add chart descriptions --- elasticsearch/Chart.yaml | 2 +- elasticsearch/README.md | 2 +- filebeat/Chart.yaml | 2 +- filebeat/README.md | 2 +- kibana/Chart.yaml | 2 +- kibana/README.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml index ad42aa16b..93484ed7e 100755 --- a/elasticsearch/Chart.yaml +++ b/elasticsearch/Chart.yaml @@ -1,4 +1,4 @@ -description: Elasticsearch +description: Official Elastic helm chart for Elasticsearch home: https://github.com/elastic/helm-charts maintainers: - email: helm-charts@elastic.co diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 650163f4d..a119ea5f0 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -1,6 +1,6 @@ # Elasticsearch Helm Chart -This functionality is in beta status and may be changed or removed completely in a future release. Elastic will take a best effort approach to fix any issues, but beta features are not subject to the support SLA of official GA features. +This functionality is in beta and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features. This helm chart is a lightweight way to configure and run our official [Elasticsearch docker image](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html) diff --git a/filebeat/Chart.yaml b/filebeat/Chart.yaml index 1137487c6..2d5f898a9 100755 --- a/filebeat/Chart.yaml +++ b/filebeat/Chart.yaml @@ -1,4 +1,4 @@ -description: Filebeat +description: Official Elastic helm chart for Filebeat home: https://github.com/elastic/helm-charts maintainers: - email: helm-charts@elastic.co diff --git a/filebeat/README.md b/filebeat/README.md index ba271f7da..f1f86e089 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -1,6 +1,6 @@ # Filebeat Helm Chart -This functionality is in beta status and may be changed or removed completely in a future release. Elastic will take a best effort approach to fix any issues, but beta features are not subject to the support SLA of official GA features. +This functionality is in beta and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features. This helm chart is a lightweight way to configure and run our official [Filebeat docker image](https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html). diff --git a/kibana/Chart.yaml b/kibana/Chart.yaml index 9c538a3d7..cd5a4d1dc 100755 --- a/kibana/Chart.yaml +++ b/kibana/Chart.yaml @@ -1,4 +1,4 @@ -description: Kibana +description: Official Elastic helm chart for Kibana home: https://github.com/elastic/helm-charts maintainers: - email: helm-charts@elastic.co diff --git a/kibana/README.md b/kibana/README.md index 09184cabe..eec510ca9 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -1,6 +1,6 @@ # Kibana Helm Chart -This functionality is in beta status and may be changed or removed completely in a future release. Elastic will take a best effort approach to fix any issues, but beta features are not subject to the support SLA of official GA features. +This functionality is in beta and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features. This helm chart is a lightweight way to configure and run our official [Kibana docker image](https://www.elastic.co/guide/en/kibana/current/docker.html) From 0b866d518d1a1bce7385618b5837ccb04b6d0e53 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Wed, 12 Jun 2019 15:52:39 +0200 Subject: [PATCH 024/115] [kibana] Add configurable nodePort to service spec Fixes: #139 --- kibana/README.md | 2 +- kibana/templates/service.yaml | 3 +++ kibana/tests/kibana_test.py | 18 +++++++++++++++++- kibana/values.yaml | 1 + 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/kibana/README.md b/kibana/README.md index eec510ca9..2d2415922 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -65,7 +65,7 @@ helm install --name kibana elastic/kibana --version 7.1.1 --set imageTag=7.1.1 | `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) so that you can target specific nodes for your Kibana instances | `{}` | | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | | `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Kibana service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | -| `service` | Configurable [service](https://kubernetes.io/docs/concepts/services-networking/service/) to expose the Kibana service. See [`values.yaml`](./values.yaml) for an example | `type: ClusterIP`
`port: 5601`
`annotations: {}` | +| `service` | Configurable [service](https://kubernetes.io/docs/concepts/services-networking/service/) to expose the Kibana service. See [`values.yaml`](./values.yaml) for an example | `type: ClusterIP`
`port: 5601`
`nodePort:`
`annotations: {}` | ## Examples diff --git a/kibana/templates/service.yaml b/kibana/templates/service.yaml index 86dda2f3a..60f47eb82 100644 --- a/kibana/templates/service.yaml +++ b/kibana/templates/service.yaml @@ -15,6 +15,9 @@ spec: type: {{ .Values.service.type }} ports: - port: {{ .Values.service.port }} +{{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} +{{- end }} protocol: TCP name: http targetPort: {{ .Values.httpPort }} diff --git a/kibana/tests/kibana_test.py b/kibana/tests/kibana_test.py index defc23022..fd07c3fc6 100644 --- a/kibana/tests/kibana_test.py +++ b/kibana/tests/kibana_test.py @@ -274,4 +274,20 @@ def test_service_annotatations(): ''' r = helm_template(config) s = r['service'][name]['metadata']['annotations']['service.beta.kubernetes.io/aws-load-balancer-internal'] - assert s == "0.0.0.0/0" \ No newline at end of file + assert s == "0.0.0.0/0" + +def test_adding_a_nodePort(): + config = '' + + r = helm_template(config) + + assert 'nodePort' not in r['service'][name]['spec']['ports'][0] + + config = ''' + service: + nodePort: 30001 + ''' + + r = helm_template(config) + + assert r['service'][name]['spec']['ports'][0]['nodePort'] == 30001 diff --git a/kibana/values.yaml b/kibana/values.yaml index d3a212f57..6cf1bb47e 100755 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -74,6 +74,7 @@ updateStrategy: service: type: ClusterIP port: 5601 + nodePort: annotations: {} # cloud.google.com/load-balancer-type: "Internal" # service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 From 22c3440d62996393d94c11cffd72a7836efd3ed2 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Wed, 12 Jun 2019 16:31:59 +0200 Subject: [PATCH 025/115] [elasticsearch] Add configurable nodePort to service spec Closes: #157 --- elasticsearch/README.md | 3 ++- elasticsearch/templates/service.yaml | 3 +++ elasticsearch/tests/elasticsearch_test.py | 17 +++++++++++++++++ elasticsearch/values.yaml | 1 + 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 2f3f6978c..295e4ad64 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -90,6 +90,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.1.1 --set im | `httpPort` | The http port that Kubernetes will use for the healthchecks and the service. If you change this you will also need to set [http.port](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#_settings) in `extraEnvs` | `9200` | | `transportPort` | The transport port that Kubernetes will use for the service. If you change this you will also need to set [transport port configuration](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#_transport_settings) in `extraEnvs` | `9300` | | `service.type` | Type of elasticsearch service. [Service Types](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) | `ClusterIP` | +| `service.nodePort` | Custom [nodePort](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport) port that can be set if you are using `service.type: nodePort`. | `` | | `service.annotations` | Annotations that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` [Annotations](https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws) | `{}` | | `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) for the statefulset. By default Kubernetes will wait for the cluster to be green after upgrading each pod. Setting this to `OnDelete` will allow you to manually delete each pod during upgrades | `RollingUpdate` | | `maxUnavailable` | The [maxUnavailable](https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget) value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group | `1` | @@ -102,7 +103,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.1.1 --set im | `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) so that you can target specific nodes for your Elasticsearch cluster | `{}` | | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | | `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Elasticsearch service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | -| `schedulerName` | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods) | `nil` | +| `schedulerName` | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods) | `nil` | ## Try it out diff --git a/elasticsearch/templates/service.yaml b/elasticsearch/templates/service.yaml index f488c49fb..474e8f919 100644 --- a/elasticsearch/templates/service.yaml +++ b/elasticsearch/templates/service.yaml @@ -21,6 +21,9 @@ spec: - name: http protocol: TCP port: {{ .Values.httpPort }} +{{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} +{{- end }} - name: transport protocol: TCP port: {{ .Values.transportPort }} diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 052129e76..5635c972e 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -584,3 +584,20 @@ def test_scheduler_name(): ''' r = helm_template(config) assert r['statefulset'][uname]['spec']['template']['spec']['schedulerName'] == "stork" + + +def test_adding_a_nodePort(): + config = '' + + r = helm_template(config) + + assert 'nodePort' not in r['service'][uname]['spec']['ports'][0] + + config = ''' + service: + nodePort: 30001 + ''' + + r = helm_template(config) + + assert r['service'][uname]['spec']['ports'][0]['nodePort'] == 30001 diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 774e528c9..14d28f71a 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -119,6 +119,7 @@ transportPort: 9300 service: type: ClusterIP + nodePort: annotations: {} updateStrategy: RollingUpdate From 7b862daf7f0fb68fa0c16260b57e363253b9b2a3 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Wed, 12 Jun 2019 16:40:22 +0200 Subject: [PATCH 026/115] Move retries to the docker build level to help with test flakiness Looking at the last build failure it seems like the curl --retry option doesn't retry for all kinds of failures. Adding the retry with the sleep will hopefully make this not flake occasionally. --- helpers/common.mk | 2 +- helpers/terraform/Dockerfile | 8 ++++---- helpers/terraform/Makefile | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/helpers/common.mk b/helpers/common.mk index bd2c71881..7f38e2d41 100644 --- a/helpers/common.mk +++ b/helpers/common.mk @@ -10,7 +10,7 @@ template: build: cd ../helpers/helm-tester && \ - docker build -t helm-tester . + for i in {1..5}; do docker build -t helm-tester . && break || sleep 15; done pytest: pytest -sv --color=yes diff --git a/helpers/terraform/Dockerfile b/helpers/terraform/Dockerfile index be56f5e95..67b701f12 100644 --- a/helpers/terraform/Dockerfile +++ b/helpers/terraform/Dockerfile @@ -16,22 +16,22 @@ RUN yum -y install \ gcloud config set component_manager/disable_update_check true && \ gcloud version -RUN curl --retry 5 -O https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip && \ +RUN curl -O https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip && \ unzip vault_${VAULT_VERSION}_linux_amd64.zip -d /usr/local/bin/ && \ chmod +x /usr/local/bin/vault && \ vault version -RUN curl --retry 5 -O https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ +RUN curl -O https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin/ && \ rm -f terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ terraform version -RUN curl --retry 5 -O https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \ +RUN curl -O https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \ mv kubectl /usr/local/bin/ && \ chmod a+x /usr/local/bin/kubectl && \ kubectl version --client -RUN curl --retry 5 -O https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ +RUN curl -O https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ mv linux-amd64/helm /usr/local/bin/ && \ rm -rf linux-amd64 && \ diff --git a/helpers/terraform/Makefile b/helpers/terraform/Makefile index b460db75e..afdf700ca 100644 --- a/helpers/terraform/Makefile +++ b/helpers/terraform/Makefile @@ -66,4 +66,4 @@ integration: creds make build: - docker build -t helm-charts . + for i in {1..5}; do docker build -t helm-charts . && break || sleep 15; done From 1eac119672b97691491a4dc1e11030839fdfa227 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 17 Jun 2019 19:53:23 +0200 Subject: [PATCH 027/115] [kibana] Always set server.host to the docker default Fixes: #156 The Kibana docker image comes with [server.host](https://github.com/elastic/kibana/blob/3a1d4ad0acd986f55b7da532878e842dd62a1237/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.js#L30) set in the default `kibana.yml`. This setting is needed to make sure that services and ingress work correctly. If someone overrides the kibana.yml with custom configuration this setting changes back to the kibana default which is `localhost`. This keeps the setting consistent for the default helm install with default values and for anyone customising `kibana.yml` --- kibana/README.md | 1 + kibana/examples/default/test/goss.yaml | 6 ++++++ kibana/examples/security/security.yml | 6 ++++++ kibana/templates/deployment.yaml | 2 ++ kibana/tests/kibana_test.py | 14 ++++++++++++++ kibana/values.yaml | 2 ++ 6 files changed, 31 insertions(+) diff --git a/kibana/README.md b/kibana/README.md index 2d2415922..535dc5c83 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -50,6 +50,7 @@ helm install --name kibana elastic/kibana --version 7.1.1 --set imageTag=7.1.1 | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 2Gi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | | `protocol` | The protocol that will be used for the readinessProbe. Change this to `https` if you have `server.ssl.enabled: true` set | `http` | +| `serverHost` | The [`server.host`](https://www.elastic.co/guide/en/kibana/current/settings.html) Kibana setting. This is set explicitly so that the default always matches what comes with the docker image. | `0.0.0.0` | | `healthCheckPath` | The path used for the readinessProbe to check that Kibana is ready | `/app/kibana` | | `kibanaConfig` | Allows you to add any config files in `/usr/share/kibana/config/` such as `kibana.yml`. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | | `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `{}` | diff --git a/kibana/examples/default/test/goss.yaml b/kibana/examples/default/test/goss.yaml index 5710496ce..9c36edb4d 100644 --- a/kibana/examples/default/test/goss.yaml +++ b/kibana/examples/default/test/goss.yaml @@ -12,3 +12,9 @@ http: http://helm-kibana-default-kibana:5601/app/kibana: status: 200 timeout: 2000 + +port: + tcp:5601: + listening: true + ip: + - '0.0.0.0' diff --git a/kibana/examples/security/security.yml b/kibana/examples/security/security.yml index 3517ed8e3..9ce3bccc5 100644 --- a/kibana/examples/security/security.yml +++ b/kibana/examples/security/security.yml @@ -34,3 +34,9 @@ secretMounts: - name: kibana-certificates secretName: kibana-certificates path: /usr/share/kibana/config/certs/kibana + +port: + tcp:5601: + listening: true + ip: + - '0.0.0.0' diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml index 52a473d67..ce7b01eac 100644 --- a/kibana/templates/deployment.yaml +++ b/kibana/templates/deployment.yaml @@ -72,6 +72,8 @@ spec: - name: ELASTICSEARCH_HOSTS value: "{{ .Values.elasticsearchHosts }}" {{- end }} + - name: SERVER_HOST + value: "{{ .Values.serverHost }}" {{- if .Values.extraEnvs }} {{ toYaml .Values.extraEnvs | indent 10 }} {{- end }} diff --git a/kibana/tests/kibana_test.py b/kibana/tests/kibana_test.py index fd07c3fc6..b802445f7 100644 --- a/kibana/tests/kibana_test.py +++ b/kibana/tests/kibana_test.py @@ -31,6 +31,9 @@ def test_defaults(): assert c['env'][0]['name'] == 'ELASTICSEARCH_HOSTS' assert c['env'][0]['value'] == elasticsearchHosts + assert c['env'][1]['name'] == 'SERVER_HOST' + assert c['env'][1]['value'] == '0.0.0.0' + assert 'http "/app/kibana"' in c['readinessProbe']['exec']['command'][-1] # Empty customizable defaults @@ -291,3 +294,14 @@ def test_adding_a_nodePort(): r = helm_template(config) assert r['service'][name]['spec']['ports'][0]['nodePort'] == 30001 + +def test_override_the_serverHost(): + config = ''' + serverHost: "localhost" + ''' + + r = helm_template(config) + + c = r['deployment'][name]['spec']['template']['spec']['containers'][0] + assert c['env'][1]['name'] == 'SERVER_HOST' + assert c['env'][1]['value'] == 'localhost' diff --git a/kibana/values.yaml b/kibana/values.yaml index 6cf1bb47e..4075f7efd 100755 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -34,6 +34,8 @@ resources: protocol: http +serverHost: "0.0.0.0" + healthCheckPath: "/app/kibana" # Allows you to add any config files in /usr/share/kibana/config/ From f48b3aa56c808aefc5233f7894bb3afc29853af3 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 17 Jun 2019 20:59:48 +0200 Subject: [PATCH 028/115] [elasticsearch] Update security example docs to match reality Over time the security example changed a bit and these instructions were no longer valid or working. Fixes: #166 --- elasticsearch/README.md | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 295e4ad64..e1c5955a9 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -129,33 +129,29 @@ make ### Security -A cluster with X-Pack security enabled +A cluster with security enabled -* Generate SSL certificates following the [official docs](https://www.elastic.co/guide/en/elasticsearch/reference/6.7/configuring-tls.html#node-certificates) -* Create Kubernetes secrets for authentication credentials and certificates +* Generate SSL certificates following the [official docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-tls.html#node-certificates) +* Create Kubernetes secrets for authentication credentials and certificates. Replace `$YOUR_SECRET_PASSWORD` with your own password. ``` - kubectl create secret generic elastic-credentials --from-literal=password=changeme --from-literal=username=elastic + kubectl create secret generic elastic-credentials --from-literal=password=$YOUR_SECRET_PASSWORD --from-literal=username=elastic kubectl create secret generic elastic-certificates --from-file=elastic-certificates.p12 ``` * Deploy! ``` cd examples/security - make + helm upgrade --wait --timeout=600 --install --values ./security.yml elasticsearch ../../ ``` * Attach into one of the containers ``` - kubectl exec -ti $(kubectl get pods -l release=helm-es-security -o name | awk -F'/' '{ print $NF }' | head -n 1) bash + kubectl exec -ti $(kubectl get --no-headers=true pods -l release=elasticsearch -o custom-columns=:metadata.name | head -n 1 ) bash ``` * Test that authentication is now enabled ``` - curl 'http://localhost:9200/' # This one will fail - curl -u elastic:changeme 'http://localhost:9200/' - ``` -* Install some test data to play around with - ``` - wget https://download.elastic.co/demos/kibana/gettingstarted/logs.jsonl.gz && gunzip logs.jsonl.gz && curl -u elastic:changeme -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/_bulk?pretty' --data-binary @logs.jsonl + curl -k 'https://localhost:9200/' # This one will fail + curl -k -u $ELASTIC_USERNAME:$ELASTIC_PASSWORD https://localhost:9200/ ``` ### FAQ From 2968c3f6381b60ec4188f26a7cc2b157935c0a29 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 17 Jun 2019 20:09:38 +0200 Subject: [PATCH 029/115] Add GKE 1.13 to automated testing suite --- helpers/matrix.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/helpers/matrix.yml b/helpers/matrix.yml index 2e43ca6ad..2a9744c46 100644 --- a/helpers/matrix.yml +++ b/helpers/matrix.yml @@ -24,3 +24,4 @@ FILEBEAT_SUITE: KUBERNETES_VERSION: - '1.11' - '1.12' + - '1.13' From b17957b67a74f18a2c7000a191a118d4d05877b9 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 18 Jun 2019 19:23:27 +0200 Subject: [PATCH 030/115] Bump timeouts for GKE cluster operations to 3 hours The previous PR test timed out after 30 minutes even though it normally takes only around 10 minutes. The cluster was actually created in the end but it was delayed due to current GCP issues. --- helpers/terraform/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helpers/terraform/main.tf b/helpers/terraform/main.tf index 635d9f436..5136b609e 100644 --- a/helpers/terraform/main.tf +++ b/helpers/terraform/main.tf @@ -25,8 +25,8 @@ resource "google_container_cluster" "cluster" { } timeouts { - create = "30m" - delete = "30m" - update = "30m" + create = "180m" + delete = "180m" + update = "180m" } } From 1292370505e18ff286c230a912089c3c62ffe4e8 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Wed, 19 Jun 2019 08:17:12 +0200 Subject: [PATCH 031/115] Remove goss port test and instead do http calls on 0.0.0.0 and service With the default install on GKE 1.13 the default bound port is now ipv4 instead of ipv6. There is an open issue in goss https://github.com/aelsabbahy/goss/issues/149 to allow testing for situations like this where it is listening on both ports. However the only important thing to test is to make sure that this this port is listening publicly and that the service actually works. Also switched the security example to test against the service to make sure we don't hit the same kibana bug as in #156 --- elasticsearch/examples/default/test/goss.yaml | 10 ++-------- elasticsearch/examples/security/test/goss.yaml | 6 +++--- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/elasticsearch/examples/default/test/goss.yaml b/elasticsearch/examples/default/test/goss.yaml index 5d1613971..353212719 100644 --- a/elasticsearch/examples/default/test/goss.yaml +++ b/elasticsearch/examples/default/test/goss.yaml @@ -1,15 +1,9 @@ -port: - tcp6:9200: - listening: true - ip: - - '::' - kernel-param: vm.max_map_count: value: '262144' http: - http://localhost:9200/_cluster/health: + http://elasticsearch-master:9200/_cluster/health: status: 200 timeout: 2000 body: @@ -17,7 +11,7 @@ http: - '"number_of_nodes":3' - '"number_of_data_nodes":3' - http://localhost:9200: + http://0.0.0.0:9200: status: 200 timeout: 2000 body: diff --git a/elasticsearch/examples/security/test/goss.yaml b/elasticsearch/examples/security/test/goss.yaml index a2777635c..9bd0ed6c7 100644 --- a/elasticsearch/examples/security/test/goss.yaml +++ b/elasticsearch/examples/security/test/goss.yaml @@ -1,5 +1,5 @@ http: - https://localhost:9200/_cluster/health: + https://security-master:9200/_cluster/health: status: 200 timeout: 2000 allow-insecure: true @@ -10,7 +10,7 @@ http: - '"number_of_nodes":3' - '"number_of_data_nodes":3' - https://localhost:9200/: + https://0.0.0.0:9200/: status: 200 timeout: 2000 allow-insecure: true @@ -21,7 +21,7 @@ http: - '"name" : "security-master-0"' - 'You Know, for Search' - https://localhost:9200/_xpack/license: + https://0.0.0.0:9200/_xpack/license: status: 200 timeout: 2000 allow-insecure: true From 54aa7b2c0444715d1cfc8421baf21d8b9fbc6e36 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Wed, 19 Jun 2019 09:47:26 +0200 Subject: [PATCH 032/115] Drop support for 5.x This actually should have been removed during the 7.0 release. It totally still works but won't be actively tested anymore. Since 5.x uses the same zen discovery configuration as 6.x it will likely stay working for the whole lifecycle of 6.x too. --- elasticsearch/README.md | 6 +++--- elasticsearch/examples/5.x/Makefile | 20 -------------------- elasticsearch/examples/5.x/test/goss.yaml | 21 --------------------- elasticsearch/examples/5.x/values.yaml | 17 ----------------- filebeat/README.md | 6 +++--- helpers/bumper.py | 1 - helpers/matrix.yml | 2 -- kibana/README.md | 6 +++--- kibana/examples/5.x/Makefile | 12 ------------ kibana/examples/5.x/test/goss.yaml | 14 -------------- kibana/examples/5.x/values.yml | 16 ---------------- 11 files changed, 9 insertions(+), 112 deletions(-) delete mode 100644 elasticsearch/examples/5.x/Makefile delete mode 100644 elasticsearch/examples/5.x/test/goss.yaml delete mode 100644 elasticsearch/examples/5.x/values.yaml delete mode 100644 kibana/examples/5.x/Makefile delete mode 100644 kibana/examples/5.x/test/goss.yaml delete mode 100644 kibana/examples/5.x/values.yml diff --git a/elasticsearch/README.md b/elasticsearch/README.md index e1c5955a9..a17ea420d 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -41,9 +41,9 @@ If you currently have a cluster deployed with the [helm/charts stable](https://g This chart is tested with the latest supported versions. The currently tested versions are: -| 5.x | 6.x | 7.x | -| ------ | ----- | ----- | -| 5.6.16 | 6.8.0 | 7.1.1 | +| 6.x | 7.x | +| ----- | ----- | +| 6.8.0 | 7.1.1 | Examples of installing older major versions can be found in the [examples](./examples) directory. diff --git a/elasticsearch/examples/5.x/Makefile b/elasticsearch/examples/5.x/Makefile deleted file mode 100644 index 5c58bfed3..000000000 --- a/elasticsearch/examples/5.x/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -default: test - -include ../../../helpers/examples.mk - -RELEASE := helm-es-fivex - -install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ ; \ - -restart: - helm upgrade --set terminationGracePeriod=121 --wait --timeout=600 --install $(RELEASE) ../../ ; \ - -purge: - helm del --purge $(RELEASE) - -secrets: - kubectl delete secrets elastic-fivex-credentials || true - kubectl create secret generic elastic-fivex-credentials --from-literal=password=changeme --from-literal=username=elastic - -test: secrets install goss diff --git a/elasticsearch/examples/5.x/test/goss.yaml b/elasticsearch/examples/5.x/test/goss.yaml deleted file mode 100644 index f9afdeb74..000000000 --- a/elasticsearch/examples/5.x/test/goss.yaml +++ /dev/null @@ -1,21 +0,0 @@ -http: - http://localhost:9200/_cluster/health: - status: 200 - timeout: 2000 - username: '{{ .Env.ELASTIC_USERNAME }}' - password: '{{ .Env.ELASTIC_PASSWORD }}' - body: - - 'green' - - '"number_of_nodes":3' - - '"number_of_data_nodes":3' - - http://localhost:9200/: - status: 200 - timeout: 2000 - username: '{{ .Env.ELASTIC_USERNAME }}' - password: '{{ .Env.ELASTIC_PASSWORD }}' - body: - - '"number" : "5.6.16"' - - '"cluster_name" : "fivex"' - - '"name" : "fivex-master-0"' - - 'You Know, for Search' diff --git a/elasticsearch/examples/5.x/values.yaml b/elasticsearch/examples/5.x/values.yaml deleted file mode 100644 index 6736b650e..000000000 --- a/elasticsearch/examples/5.x/values.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- - -clusterName: "fivex" -imageTag: "5.6.16" -esMajorVersion: 5 - -extraEnvs: - - name: ELASTIC_PASSWORD - valueFrom: - secretKeyRef: - name: elastic-fivex-credentials - key: password - - name: ELASTIC_USERNAME - valueFrom: - secretKeyRef: - name: elastic-fivex-credentials - key: username diff --git a/filebeat/README.md b/filebeat/README.md index f1f86e089..7fb52a527 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -28,9 +28,9 @@ This helm chart is a lightweight way to configure and run our official [Filebeat This chart is tested with the latest supported versions. The currently tested versions are: -| 5.x | 6.x | 7.x | -| ------ | ----- | ----- | -| 5.6.16 | 6.8.0 | 7.1.1 | +| 6.x | 7.x | +| ----- | ----- | +| 6.8.0 | 7.1.1 | Examples of installing older major versions can be found in the [examples](./examples) directory. diff --git a/helpers/bumper.py b/helpers/bumper.py index d08709f34..30ebd018c 100755 --- a/helpers/bumper.py +++ b/helpers/bumper.py @@ -8,7 +8,6 @@ os.chdir(os.path.join(os.path.dirname(__file__), '..')) versions = { - 5: '5.6.16', 6: '6.8.0', 7: '7.1.1', } diff --git a/helpers/matrix.yml b/helpers/matrix.yml index 2a9744c46..fbe55177d 100644 --- a/helpers/matrix.yml +++ b/helpers/matrix.yml @@ -8,13 +8,11 @@ ES_SUITE: - oss - security - upgrade - - 5.x - 6.x KIBANA_SUITE: - default - oss - security - - 5.x - 6.x FILEBEAT_SUITE: - default diff --git a/kibana/README.md b/kibana/README.md index 535dc5c83..b479d6a2c 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -24,9 +24,9 @@ This helm chart is a lightweight way to configure and run our official [Kibana d This chart is tested with the latest supported versions. The currently tested versions are: -| 5.x | 6.x | 7.x | -| ------ | ----- | ----- | -| 5.6.16 | 6.8.0 | 7.1.1 | +| 6.x | 7.x | +| ----- | ----- | +| 6.8.0 | 7.1.1 | Examples of installing older major versions can be found in the [examples](./examples) directory. diff --git a/kibana/examples/5.x/Makefile b/kibana/examples/5.x/Makefile deleted file mode 100644 index d7ab7579c..000000000 --- a/kibana/examples/5.x/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -default: test -include ../../../helpers/examples.mk - -RELEASE := helm-kibana-fivex - -install: - helm upgrade --wait --timeout=600 --install --values ./values.yml $(RELEASE) ../../ ; \ - -purge: - helm del --purge $(RELEASE) - -test: install goss diff --git a/kibana/examples/5.x/test/goss.yaml b/kibana/examples/5.x/test/goss.yaml deleted file mode 100644 index 0756b9f4a..000000000 --- a/kibana/examples/5.x/test/goss.yaml +++ /dev/null @@ -1,14 +0,0 @@ -http: - http://localhost:5601/api/status: - status: 200 - timeout: 2000 - body: - - '"version":"5.6.16"' - username: '{{ .Env.ELASTICSEARCH_USERNAME }}' - password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' - - http://localhost:5601/app/kibana: - status: 200 - timeout: 2000 - username: '{{ .Env.ELASTICSEARCH_USERNAME }}' - password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' diff --git a/kibana/examples/5.x/values.yml b/kibana/examples/5.x/values.yml deleted file mode 100644 index 226dd60f6..000000000 --- a/kibana/examples/5.x/values.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- - -imageTag: 5.6.16 -elasticsearchURL: "http://fivex-master:9200" - -extraEnvs: - - name: 'ELASTICSEARCH_USERNAME' - valueFrom: - secretKeyRef: - name: elastic-fivex-credentials - key: username - - name: 'ELASTICSEARCH_PASSWORD' - valueFrom: - secretKeyRef: - name: elastic-fivex-credentials - key: password From b0a769e06a587e249a80c9f2d1978751afb2b511 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Thu, 20 Jun 2019 17:47:30 +0200 Subject: [PATCH 033/115] Test kibana directly on 0.0.0.0 instead of checking the port --- kibana/examples/default/test/goss.yaml | 10 ++-------- kibana/examples/security/security.yml | 6 ------ kibana/examples/security/test/goss.yaml | 2 +- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/kibana/examples/default/test/goss.yaml b/kibana/examples/default/test/goss.yaml index 9c36edb4d..5480fc54e 100644 --- a/kibana/examples/default/test/goss.yaml +++ b/kibana/examples/default/test/goss.yaml @@ -1,20 +1,14 @@ http: - http://localhost:5601/api/status: + http://0.0.0.0:5601/api/status: status: 200 timeout: 2000 body: - '"number":"7.1.1"' - http://localhost:5601/app/kibana: + http://0.0.0.0:5601/app/kibana: status: 200 timeout: 2000 http://helm-kibana-default-kibana:5601/app/kibana: status: 200 timeout: 2000 - -port: - tcp:5601: - listening: true - ip: - - '0.0.0.0' diff --git a/kibana/examples/security/security.yml b/kibana/examples/security/security.yml index 9ce3bccc5..3517ed8e3 100644 --- a/kibana/examples/security/security.yml +++ b/kibana/examples/security/security.yml @@ -34,9 +34,3 @@ secretMounts: - name: kibana-certificates secretName: kibana-certificates path: /usr/share/kibana/config/certs/kibana - -port: - tcp:5601: - listening: true - ip: - - '0.0.0.0' diff --git a/kibana/examples/security/test/goss.yaml b/kibana/examples/security/test/goss.yaml index 51b3ce214..6ac68a8c2 100644 --- a/kibana/examples/security/test/goss.yaml +++ b/kibana/examples/security/test/goss.yaml @@ -1,5 +1,5 @@ http: - https://localhost:5601/app/kibana: + https://0.0.0.0:5601/app/kibana: status: 200 timeout: 2000 allow-insecure: true From 9dd7bbe2d6bfb113bd63d313063f05c3a024f4dd Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Fri, 21 Jun 2019 09:15:09 +0200 Subject: [PATCH 034/115] Always run tests against localhost or the service endpoint This makes sure that: 1. The tests work in ipv4 and ipv6 environments 2. Testing the service endpoint makes sure that the listening address is properly configured to allow traffic Addresses: https://github.com/elastic/helm-charts/pull/169#pullrequestreview-252414635 --- elasticsearch/examples/default/test/goss.yaml | 2 +- elasticsearch/examples/security/test/goss.yaml | 4 ++-- kibana/examples/default/test/goss.yaml | 4 ++-- kibana/examples/security/test/goss.yaml | 9 ++++++++- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/elasticsearch/examples/default/test/goss.yaml b/elasticsearch/examples/default/test/goss.yaml index 353212719..93fdaad82 100644 --- a/elasticsearch/examples/default/test/goss.yaml +++ b/elasticsearch/examples/default/test/goss.yaml @@ -11,7 +11,7 @@ http: - '"number_of_nodes":3' - '"number_of_data_nodes":3' - http://0.0.0.0:9200: + http://localhost:9200: status: 200 timeout: 2000 body: diff --git a/elasticsearch/examples/security/test/goss.yaml b/elasticsearch/examples/security/test/goss.yaml index 9bd0ed6c7..c6d4b987b 100644 --- a/elasticsearch/examples/security/test/goss.yaml +++ b/elasticsearch/examples/security/test/goss.yaml @@ -10,7 +10,7 @@ http: - '"number_of_nodes":3' - '"number_of_data_nodes":3' - https://0.0.0.0:9200/: + https://localhost:9200/: status: 200 timeout: 2000 allow-insecure: true @@ -21,7 +21,7 @@ http: - '"name" : "security-master-0"' - 'You Know, for Search' - https://0.0.0.0:9200/_xpack/license: + https://localhost:9200/_xpack/license: status: 200 timeout: 2000 allow-insecure: true diff --git a/kibana/examples/default/test/goss.yaml b/kibana/examples/default/test/goss.yaml index 5480fc54e..5710496ce 100644 --- a/kibana/examples/default/test/goss.yaml +++ b/kibana/examples/default/test/goss.yaml @@ -1,11 +1,11 @@ http: - http://0.0.0.0:5601/api/status: + http://localhost:5601/api/status: status: 200 timeout: 2000 body: - '"number":"7.1.1"' - http://0.0.0.0:5601/app/kibana: + http://localhost:5601/app/kibana: status: 200 timeout: 2000 diff --git a/kibana/examples/security/test/goss.yaml b/kibana/examples/security/test/goss.yaml index 6ac68a8c2..caf26e40f 100644 --- a/kibana/examples/security/test/goss.yaml +++ b/kibana/examples/security/test/goss.yaml @@ -1,5 +1,12 @@ http: - https://0.0.0.0:5601/app/kibana: + https://localhost:5601/app/kibana: + status: 200 + timeout: 2000 + allow-insecure: true + username: '{{ .Env.ELASTICSEARCH_USERNAME }}' + password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' + + https://helm-kibana-security-kibana:5601/app/kibana: status: 200 timeout: 2000 allow-insecure: true From 0341fd6b7b6aabe255fa47e86a159b9e0119119d Mon Sep 17 00:00:00 2001 From: Nhat Hoang Date: Fri, 21 Jun 2019 14:22:57 +0200 Subject: [PATCH 035/115] change secretName field --- filebeat/templates/daemonset.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index bc53b51ba..cf4aad709 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -42,7 +42,7 @@ spec: {{- range .Values.secretMounts }} - name: {{ .name }} secret: - secretName: {{ .name }} + secretName: {{ .secretName }} {{- end }} {{- if .Values.filebeatConfig }} - name: filebeat-config @@ -65,7 +65,8 @@ spec: {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: -{{ toYaml .Values.imagePullSecrets | indent 8 }} +{{ toYaml .Values.imagePull +s | indent 8 }} {{- end }} containers: - name: "filebeat" From f2c510b7ce2d110ac8382ed43a17cf57c08d80dd Mon Sep 17 00:00:00 2001 From: Naseem Date: Mon, 17 Jun 2019 18:52:27 -0400 Subject: [PATCH 036/115] Run as 1000 Signed-off-by: Naseem --- kibana/README.md | 3 ++- kibana/templates/deployment.yaml | 8 ++++---- kibana/values.yaml | 13 +++++++++++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/kibana/README.md b/kibana/README.md index 2d2415922..c60b38678 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -52,7 +52,8 @@ helm install --name kibana elastic/kibana --version 7.1.1 --set imageTag=7.1.1 | `protocol` | The protocol that will be used for the readinessProbe. Change this to `https` if you have `server.ssl.enabled: true` set | `http` | | `healthCheckPath` | The path used for the readinessProbe to check that Kibana is ready | `/app/kibana` | | `kibanaConfig` | Allows you to add any config files in `/usr/share/kibana/config/` such as `kibana.yml`. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | -| `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `{}` | +| `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000` | +| `securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) for the container | `capabilities.drop:[ALL]`
`runAsNonRoot: true`
`runAsUser: 1000` | | `serviceAccount` | Allows you to overwrite the "default" [serviceAccount](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) for the pod | `[]` | | `priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `` | | `antiAffinityTopologyKey` | The [anti-affinity topology key](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). By default this will prevent multiple Kibana instances from running on the same Kubernetes node | `kubernetes.io/hostname` | diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml index 52a473d67..af608ce1d 100644 --- a/kibana/templates/deployment.yaml +++ b/kibana/templates/deployment.yaml @@ -27,10 +27,8 @@ spec: {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end }} -{{- if .Values.podSecurityContext }} securityContext: {{ toYaml .Values.podSecurityContext | indent 8 }} -{{- end }} {{- if .Values.serviceAccount }} serviceAccount: {{ .Values.serviceAccount }} {{- end }} @@ -63,6 +61,8 @@ spec: {{- end }} containers: - name: kibana + securityContext: +{{ toYaml .Values.securityContext | indent 10 }} image: "{{ .Values.image }}:{{ .Values.imageTag }}" env: {{- if .Values.elasticsearchURL }} @@ -78,7 +78,7 @@ spec: readinessProbe: {{ toYaml .Values.readinessProbe | indent 10 }} exec: - command: + command: - sh - -c - | @@ -93,7 +93,7 @@ spec: curl -k "$@" "{{ .Values.protocol }}://localhost:{{ .Values.httpPort }}${path}" } - + http "{{ .Values.healthCheckPath }}" ports: - containerPort: {{ .Values.httpPort }} diff --git a/kibana/values.yaml b/kibana/values.yaml index 6cf1bb47e..ba8fe42a6 100755 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -44,8 +44,17 @@ kibanaConfig: {} # nestedkey: value # If Pod Security Policy in use it may be required to specify security context as well as service account -podSecurityContext: {} - #runAsUser: "place the user id here" + +podSecurityContext: + fsGroup: 1000 + +securityContext: + capabilities: + drop: + - ALL + # readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 serviceAccount: "" From 89652ea43bdac6f635cfbb85f748218a2149d412 Mon Sep 17 00:00:00 2001 From: diegofernandes Date: Fri, 21 Jun 2019 11:19:08 -0300 Subject: [PATCH 037/115] fix support wildcard tls host on ingress --- kibana/templates/ingress.yaml | 8 +------- kibana/tests/kibana_test.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/kibana/templates/ingress.yaml b/kibana/templates/ingress.yaml index d4e7707c6..cd4914930 100644 --- a/kibana/templates/ingress.yaml +++ b/kibana/templates/ingress.yaml @@ -17,13 +17,7 @@ metadata: spec: {{- if .Values.ingress.tls }} tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} +{{ toYaml .Values.ingress.tls | indent 4 }} {{- end }} rules: {{- range .Values.ingress.hosts }} diff --git a/kibana/tests/kibana_test.py b/kibana/tests/kibana_test.py index b802445f7..5ad4a33b1 100644 --- a/kibana/tests/kibana_test.py +++ b/kibana/tests/kibana_test.py @@ -159,6 +159,34 @@ def test_adding_an_ingress_rule(): assert i['rules'][0]['http']['paths'][0]['backend']['serviceName'] == name assert i['rules'][0]['http']['paths'][0]['backend']['servicePort'] == 5601 +def test_adding_an_ingress_rule_wildcard(): + config = ''' +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: nginx + path: / + hosts: + - kibana.elastic.co + tls: + - secretName: elastic-co-wildcard + hosts: + - "*.elastic.co" +''' + + r = helm_template(config) + assert name in r['ingress'] + i = r['ingress'][name]['spec'] + assert i['tls'][0]['hosts'][0] == '*.elastic.co' + assert i['tls'][0]['secretName'] == 'elastic-co-wildcard' + + assert i['rules'][0]['host'] == 'kibana.elastic.co' + assert i['rules'][0]['http']['paths'][0]['path'] == '/' + assert i['rules'][0]['http']['paths'][0]['backend']['serviceName'] == name + assert i['rules'][0]['http']['paths'][0]['backend']['servicePort'] == 5601 + + + def test_override_the_default_update_strategy(): config = ''' updateStrategy: From 2d309cbfb25ca0f5d3c3633445144d2c5ed1de54 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 24 Jun 2019 15:45:32 +0200 Subject: [PATCH 038/115] [elasticsearch] Fix pvc annotations with multiple fields Fixes: #183 This worked just fine if there was only 1 annotation being added. When adding multiple annotations the extra whitespace was causing it to break. --- elasticsearch/templates/statefulset.yaml | 2 +- elasticsearch/tests/elasticsearch_test.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 0c5444094..be2d44f8e 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -23,7 +23,7 @@ spec: name: {{ template "uname" . }} {{- with .Values.persistence.annotations }} annotations: - {{ toYaml . | indent 4 }} +{{ toYaml . | indent 8 }} {{- end }} spec: {{ toYaml .Values.volumeClaimTemplate | indent 6 }} diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 5635c972e..9422858a2 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -343,7 +343,20 @@ def test_adding_storageclass_annotation_to_volumeclaimtemplate(): ''' r = helm_template(config) annotations = r['statefulset'][uname]['spec']['volumeClaimTemplates'][0]['metadata']['annotations'] - assert {'volume.beta.kubernetes.io/storage-class': 'id'} == annotations + assert annotations['volume.beta.kubernetes.io/storage-class'] == 'id' + +def test_adding_multiple_persistence_annotations(): + config = ''' + persistence: + annotations: + hello: world + world: hello + ''' + r = helm_template(config) + annotations = r['statefulset'][uname]['spec']['volumeClaimTemplates'][0]['metadata']['annotations'] + + assert annotations['hello'] == 'world' + assert annotations['world'] == 'hello' def test_adding_a_secret_mount(): From cdf32c1751617f2062437c56c8fa535e7bc0c0b4 Mon Sep 17 00:00:00 2001 From: Nhat Hoang Date: Mon, 24 Jun 2019 17:14:59 +0200 Subject: [PATCH 039/115] Fix unintended new line --- filebeat/templates/daemonset.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index cf4aad709..ddb0fc4eb 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -65,8 +65,7 @@ spec: {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: -{{ toYaml .Values.imagePull -s | indent 8 }} +{{ toYaml .Values.imagePullSecrets | indent 8 }} {{- end }} containers: - name: "filebeat" From 53a1b97987582505e2a05c157515ea79dd8f0d50 Mon Sep 17 00:00:00 2001 From: Nhat Hoang Date: Mon, 24 Jun 2019 17:16:51 +0200 Subject: [PATCH 040/115] Change value for secretName --- filebeat/tests/filebeat_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index da73f54e7..f4b4f0afb 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -148,7 +148,7 @@ def test_adding_a_secret_mount(): config = ''' secretMounts: - name: elastic-certificates - secretName: elastic-certificates + secretName: elastic-certs path: /usr/share/filebeat/config/certs ''' r = helm_template(config) From a67f9e59eca86d0f95f7189c800b0a5bc6c080fd Mon Sep 17 00:00:00 2001 From: Nhat Hoang Date: Mon, 24 Jun 2019 20:56:16 +0200 Subject: [PATCH 041/115] wrong secretName value --- kibana/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml index ce7b01eac..05beb8265 100644 --- a/kibana/templates/deployment.yaml +++ b/kibana/templates/deployment.yaml @@ -38,7 +38,7 @@ spec: {{- range .Values.secretMounts }} - name: {{ .name }} secret: - secretName: {{ .name }} + secretName: {{ .secretName }} {{- end }} {{- if .Values.kibanaConfig }} - name: kibanaconfig From e2cffada9548e71179a00d9f37519b94caa52224 Mon Sep 17 00:00:00 2001 From: Nhat Hoang Date: Mon, 24 Jun 2019 20:57:44 +0200 Subject: [PATCH 042/115] Wrong secretName value --- elasticsearch/templates/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 0c5444094..c87026e5e 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -96,7 +96,7 @@ spec: {{- range .Values.secretMounts }} - name: {{ .name }} secret: - secretName: {{ .name }} + secretName: {{ .secretName }} {{- end }} {{- if .Values.esConfig }} - name: esconfig From 8eef0a95ec85a7c007697334ba4ac481d2096630 Mon Sep 17 00:00:00 2001 From: Nhat Hoang Date: Mon, 24 Jun 2019 20:59:39 +0200 Subject: [PATCH 043/115] Wrong value of secretName --- elasticsearch/tests/elasticsearch_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 5635c972e..d2404dee4 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -350,7 +350,7 @@ def test_adding_a_secret_mount(): config = ''' secretMounts: - name: elastic-certificates - secretName: elastic-certificates + secretName: elastic-certs path: /usr/share/elasticsearch/config/certs ''' r = helm_template(config) @@ -362,7 +362,7 @@ def test_adding_a_secret_mount(): assert s['volumes'] == [{ 'name': 'elastic-certificates', 'secret': { - 'secretName': 'elastic-certificates' + 'secretName': 'elastic-certs' } }] @@ -371,7 +371,7 @@ def test_adding_a_secret_mount_with_subpath(): config = ''' secretMounts: - name: elastic-certificates - secretName: elastic-certificates + secretName: elastic-certs path: /usr/share/elasticsearch/config/certs subPath: cert.crt ''' From 61a2f5eec2b6b31e14104c73992273465dbfe850 Mon Sep 17 00:00:00 2001 From: Nhat Hoang Date: Mon, 24 Jun 2019 22:15:52 +0200 Subject: [PATCH 044/115] Fix filebeat_test with secretName --- filebeat/tests/filebeat_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index f4b4f0afb..0218fa6b8 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -160,7 +160,7 @@ def test_adding_a_secret_mount(): assert s['volumes'][0] == { 'name': 'elastic-certificates', 'secret': { - 'secretName': 'elastic-certificates' + 'secretName': 'elastic-certs' } } From 93997220a742faf1cbd31c50f7858a922aa20122 Mon Sep 17 00:00:00 2001 From: cclauss Date: Wed, 26 Jun 2019 16:59:04 +0200 Subject: [PATCH 045/115] Fix octal literal to work in both Python 2 and Python 3 $ __python3 -c "0600"__ ``` File "", line 1 0600 ^ SyntaxError: invalid token ``` --- filebeat/tests/filebeat_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index da73f54e7..616144d86 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -137,7 +137,7 @@ def test_adding_in_filebeat_config(): d = r['daemonset'][name]['spec']['template']['spec'] - assert {'configMap': {'name': name + '-config', 'defaultMode': 0600}, 'name': project + '-config'} in d['volumes'] + assert {'configMap': {'name': name + '-config', 'defaultMode': 0o600}, 'name': project + '-config'} in d['volumes'] assert {'mountPath': '/usr/share/filebeat/filebeat.yml', 'name': project + '-config', 'subPath': 'filebeat.yml', 'readOnly': True} in d['containers'][0]['volumeMounts'] assert {'mountPath': '/usr/share/filebeat/other-config.yml', 'name': project + '-config', 'subPath': 'other-config.yml', 'readOnly': True} in d['containers'][0]['volumeMounts'] From ce8c1b2c0b80a5cb182b588d6eedc606b4a88ae6 Mon Sep 17 00:00:00 2001 From: nathan patel Date: Wed, 26 Jun 2019 17:21:28 +0100 Subject: [PATCH 046/115] Add resources to sidecar container. Some K8s clusters have requirements that all containers contain resource definitions. This provides the option to add resource limits to sidecar containers. --- elasticsearch/templates/statefulset.yaml | 2 ++ elasticsearch/tests/elasticsearch_test.py | 24 +++++++++++++++++++++++ elasticsearch/values.yaml | 8 ++++++++ 3 files changed, 34 insertions(+) diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index be2d44f8e..683ddb83a 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -263,6 +263,8 @@ spec: sleep infinity & wait $! + resources: +{{ toYaml .Values.sidecarResources | indent 10 }} env: - name: NODE_NAME valueFrom: diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 9422858a2..c65f6174f 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -459,6 +459,30 @@ def test_adding_resources_to_initcontainer(): } } +def test_adding_resources_to_sidecar_container(): + config = ''' +initResources: + limits: + cpu: "100m" + memory: "128Mi" + requests: + cpu: "100m" + memory: "128Mi" +''' + r = helm_template(config) + i = r['statefulset'][uname]['spec']['template']['spec']['initContainers'][0] + + assert i['resources'] == { + 'requests': { + 'cpu': '100m', + 'memory': '128Mi' + }, + 'limits': { + 'cpu': '100m', + 'memory': '128Mi' + } + } + def test_adding_a_node_affinity(): config = ''' nodeAffinity: diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 14d28f71a..8db4418da 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -67,6 +67,14 @@ initResources: {} # cpu: "25m" # memory: "128Mi" +sidecarResources: {} + # limits: + # cpu: "25m" + # # memory: "128Mi" + # requests: + # cpu: "25m" + # memory: "128Mi" + networkHost: "0.0.0.0" volumeClaimTemplate: From 63690215a32d2d6aa45c3a586d8fdc361d255f30 Mon Sep 17 00:00:00 2001 From: nathan patel Date: Wed, 26 Jun 2019 18:11:02 +0100 Subject: [PATCH 047/115] Update README. --- elasticsearch/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index a17ea420d..d8b21feda 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -77,6 +77,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.1.1 --set im | `esJavaOpts` | [Java options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for Elasticsearch. This is where you should configure the [jvm heap size](https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html) | `-Xmx1g -Xms1g` | | `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 2Gi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | | `initResources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the initContainer in the statefulset | {} | +| `sidecarResources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the sidecar containers in the statefulset | {} | | `networkHost` | Value for the [network.host Elasticsearch setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/network.host.html) | `0.0.0.0` | | `volumeClaimTemplate` | Configuration for the [volumeClaimTemplate for statefulsets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-storage). You will want to adjust the storage (default `30Gi`) and the `storageClassName` if you are using a different storage class | `accessModes: [ "ReadWriteOnce" ]`
`resources.requests.storage: 30Gi` | | `persistence.annotations` | Additional persistence annotations for the `volumeClaimTemplate` | `{}` | From f17997b53cda461eb876d2912032a6cce4e208b4 Mon Sep 17 00:00:00 2001 From: nathan patel Date: Wed, 26 Jun 2019 23:06:41 +0100 Subject: [PATCH 048/115] Fix test. --- elasticsearch/tests/elasticsearch_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index c65f6174f..55fcb8acc 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -461,7 +461,7 @@ def test_adding_resources_to_initcontainer(): def test_adding_resources_to_sidecar_container(): config = ''' -initResources: +sidecarResources: limits: cpu: "100m" memory: "128Mi" @@ -470,7 +470,7 @@ def test_adding_resources_to_sidecar_container(): memory: "128Mi" ''' r = helm_template(config) - i = r['statefulset'][uname]['spec']['template']['spec']['initContainers'][0] + i = r['statefulset'][uname]['spec']['template']['spec']['containers'][1] assert i['resources'] == { 'requests': { From 28176014884d79217c8b54abc57083097df78f5f Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 10:54:51 +0200 Subject: [PATCH 049/115] Release 7.2.0 --- elasticsearch/Chart.yaml | 4 ++-- elasticsearch/README.md | 10 +++++----- elasticsearch/examples/6.x/test/goss.yaml | 2 +- elasticsearch/examples/6.x/values.yaml | 2 +- elasticsearch/examples/default/test/goss.yaml | 2 +- elasticsearch/examples/oss/test/goss.yaml | 2 +- elasticsearch/examples/upgrade/test/goss.yaml | 2 +- elasticsearch/values.yaml | 2 +- filebeat/Chart.yaml | 4 ++-- filebeat/README.md | 10 +++++----- filebeat/examples/6.x/test/goss.yaml | 2 +- filebeat/examples/6.x/values.yaml | 2 +- filebeat/examples/default/test/goss.yaml | 4 ++-- filebeat/examples/oss/test/goss.yaml | 2 +- filebeat/examples/security/test/goss.yaml | 2 +- filebeat/values.yaml | 2 +- helpers/bumper.py | 4 ++-- kibana/Chart.yaml | 4 ++-- kibana/README.md | 10 +++++----- kibana/examples/6.x/test/goss.yaml | 2 +- kibana/examples/6.x/values.yml | 2 +- kibana/examples/default/test/goss.yaml | 2 +- kibana/values.yaml | 2 +- 23 files changed, 40 insertions(+), 40 deletions(-) diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml index 93484ed7e..18077d545 100755 --- a/elasticsearch/Chart.yaml +++ b/elasticsearch/Chart.yaml @@ -4,8 +4,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: elasticsearch -version: 7.1.1 -appVersion: 7.1.1 +version: 7.2.0 +appVersion: 7.2.0 sources: - https://github.com/elastic/elasticsearch icon: https://helm.elastic.co/icons/elasticsearch.png diff --git a/elasticsearch/README.md b/elasticsearch/README.md index a17ea420d..1400fd681 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -34,7 +34,7 @@ If you currently have a cluster deployed with the [helm/charts stable](https://g ``` * Install it ``` - helm install --name elasticsearch elastic/elasticsearch --version 7.1.1 + helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 ``` ## Compatibility @@ -43,14 +43,14 @@ This chart is tested with the latest supported versions. The currently tested ve | 6.x | 7.x | | ----- | ----- | -| 6.8.0 | 7.1.1 | +| 6.8.1 | 7.2.0 | Examples of installing older major versions can be found in the [examples](./examples) directory. -While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.1.1` of Elasticsearch it would look like this: +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.2.0` of Elasticsearch it would look like this: ``` -helm install --name elasticsearch elastic/elasticsearch --version 7.1.1 --set imageTag=7.1.1 +helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 --set imageTag=7.2.0 ``` ## Configuration @@ -71,7 +71,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.1.1 --set im | `extraInitContainers` | Additional init containers to be passed to the `tpl` function | | | `secretMounts` | Allows you easily mount a secret as a file inside the statefulset. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | | `image` | The Elasticsearch docker image | `docker.elastic.co/elasticsearch/elasticsearch` | -| `imageTag` | The Elasticsearch docker image tag | `7.1.1` | +| `imageTag` | The Elasticsearch docker image tag | `7.2.0` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Elasticsearch pods | `{}` | | `esJavaOpts` | [Java options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for Elasticsearch. This is where you should configure the [jvm heap size](https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html) | `-Xmx1g -Xms1g` | diff --git a/elasticsearch/examples/6.x/test/goss.yaml b/elasticsearch/examples/6.x/test/goss.yaml index f06a0853e..cf6ea4200 100644 --- a/elasticsearch/examples/6.x/test/goss.yaml +++ b/elasticsearch/examples/6.x/test/goss.yaml @@ -11,7 +11,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "6.8.0"' + - '"number" : "6.8.1"' - '"cluster_name" : "six"' - '"name" : "six-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/examples/6.x/values.yaml b/elasticsearch/examples/6.x/values.yaml index 67bc5d833..d721bdf69 100644 --- a/elasticsearch/examples/6.x/values.yaml +++ b/elasticsearch/examples/6.x/values.yaml @@ -1,5 +1,5 @@ --- clusterName: "six" -imageTag: "6.8.0" +imageTag: "6.8.1" esMajorVersion: 6 diff --git a/elasticsearch/examples/default/test/goss.yaml b/elasticsearch/examples/default/test/goss.yaml index 93fdaad82..01ae5af40 100644 --- a/elasticsearch/examples/default/test/goss.yaml +++ b/elasticsearch/examples/default/test/goss.yaml @@ -15,7 +15,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "7.1.1"' + - '"number" : "7.2.0"' - '"cluster_name" : "elasticsearch"' - '"name" : "elasticsearch-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/examples/oss/test/goss.yaml b/elasticsearch/examples/oss/test/goss.yaml index 1169bd730..63937ec96 100644 --- a/elasticsearch/examples/oss/test/goss.yaml +++ b/elasticsearch/examples/oss/test/goss.yaml @@ -11,7 +11,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "7.1.1"' + - '"number" : "7.2.0"' - '"cluster_name" : "oss"' - '"name" : "oss-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/examples/upgrade/test/goss.yaml b/elasticsearch/examples/upgrade/test/goss.yaml index e5452a89a..b48364c03 100644 --- a/elasticsearch/examples/upgrade/test/goss.yaml +++ b/elasticsearch/examples/upgrade/test/goss.yaml @@ -11,7 +11,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "7.1.1"' + - '"number" : "7.2.0"' - '"cluster_name" : "upgrade"' - '"name" : "upgrade-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 14d28f71a..2c9472196 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -43,7 +43,7 @@ secretMounts: [] # path: /usr/share/elasticsearch/config/certs image: "docker.elastic.co/elasticsearch/elasticsearch" -imageTag: "7.1.1" +imageTag: "7.2.0" imagePullPolicy: "IfNotPresent" podAnnotations: {} diff --git a/filebeat/Chart.yaml b/filebeat/Chart.yaml index 2d5f898a9..7b30bac13 100755 --- a/filebeat/Chart.yaml +++ b/filebeat/Chart.yaml @@ -4,8 +4,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: filebeat -version: 7.1.1 -appVersion: 7.1.1 +version: 7.2.0 +appVersion: 7.2.0 sources: - https://github.com/elastic/beats icon: https://helm.elastic.co/icons/filebeat.png diff --git a/filebeat/README.md b/filebeat/README.md index 7fb52a527..fe547488c 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -21,7 +21,7 @@ This helm chart is a lightweight way to configure and run our official [Filebeat ``` * Install it ``` - helm install --name filebeat elastic/filebeat --version 7.1.1 + helm install --name filebeat elastic/filebeat --version 7.2.0 ``` ## Compatibility @@ -30,14 +30,14 @@ This chart is tested with the latest supported versions. The currently tested ve | 6.x | 7.x | | ----- | ----- | -| 6.8.0 | 7.1.1 | +| 6.8.1 | 7.2.0 | Examples of installing older major versions can be found in the [examples](./examples) directory. -While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.1.1` of Filebeat it would look like this: +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.2.0` of Filebeat it would look like this: ``` -helm install --name filebeat elastic/filebeat --version 7.1.1 --set imageTag=7.1.1 +helm install --name filebeat elastic/filebeat --version 7.2.0 --set imageTag=7.2.0 ``` @@ -50,7 +50,7 @@ helm install --name filebeat elastic/filebeat --version 7.1.1 --set imageTag=7.1 | `extraVolumes` | Any extra volumes to define for the pod | `[]` | | `hostPathRoot` | Fully-qualified [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) that will be used to persist Filebeat registry data | `/var/lib` | | `image` | The Filebeat docker image | `docker.elastic.co/beats/filebeat` | -| `imageTag` | The Filebeat docker image tag | `7.1.1` | +| `imageTag` | The Filebeat docker image tag | `7.2.0` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | | `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles. | `true` | diff --git a/filebeat/examples/6.x/test/goss.yaml b/filebeat/examples/6.x/test/goss.yaml index 045673ce1..e18f4e81c 100644 --- a/filebeat/examples/6.x/test/goss.yaml +++ b/filebeat/examples/6.x/test/goss.yaml @@ -18,4 +18,4 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-6.8.0' + - 'filebeat-6.8.1' diff --git a/filebeat/examples/6.x/values.yaml b/filebeat/examples/6.x/values.yaml index 6daa49eef..d0eeea620 100644 --- a/filebeat/examples/6.x/values.yaml +++ b/filebeat/examples/6.x/values.yaml @@ -1,4 +1,4 @@ -imageTag: 6.8.0 +imageTag: 6.8.1 extraEnvs: - name: ELASTICSEARCH_HOSTS diff --git a/filebeat/examples/default/test/goss.yaml b/filebeat/examples/default/test/goss.yaml index a06263f7d..29d4cd6df 100644 --- a/filebeat/examples/default/test/goss.yaml +++ b/filebeat/examples/default/test/goss.yaml @@ -29,7 +29,7 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-7.1.1' + - 'filebeat-7.2.0' file: /usr/share/filebeat/filebeat.yml: @@ -44,4 +44,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://elasticsearch-master:9200' - - 'version: 7.1.1' + - 'version: 7.2.0' diff --git a/filebeat/examples/oss/test/goss.yaml b/filebeat/examples/oss/test/goss.yaml index 8853288c3..a40f18579 100644 --- a/filebeat/examples/oss/test/goss.yaml +++ b/filebeat/examples/oss/test/goss.yaml @@ -19,4 +19,4 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-7.1.1' + - 'filebeat-7.2.0' diff --git a/filebeat/examples/security/test/goss.yaml b/filebeat/examples/security/test/goss.yaml index ed8a1c76b..ce3f0f708 100644 --- a/filebeat/examples/security/test/goss.yaml +++ b/filebeat/examples/security/test/goss.yaml @@ -3,7 +3,7 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-7.1.1' + - 'filebeat-7.2.0' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 1bb780110..9bb996c57 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -34,7 +34,7 @@ extraVolumes: [] hostPathRoot: /var/lib image: "docker.elastic.co/beats/filebeat" -imageTag: "7.1.1" +imageTag: "7.2.0" imagePullPolicy: "IfNotPresent" imagePullSecrets: [] diff --git a/helpers/bumper.py b/helpers/bumper.py index 30ebd018c..41a5500c9 100755 --- a/helpers/bumper.py +++ b/helpers/bumper.py @@ -8,8 +8,8 @@ os.chdir(os.path.join(os.path.dirname(__file__), '..')) versions = { - 6: '6.8.0', - 7: '7.1.1', + 6: '6.8.1', + 7: '7.2.0', } file_patterns = [ diff --git a/kibana/Chart.yaml b/kibana/Chart.yaml index cd5a4d1dc..2095e37f6 100755 --- a/kibana/Chart.yaml +++ b/kibana/Chart.yaml @@ -4,8 +4,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: kibana -version: 7.1.1 -appVersion: 7.1.1 +version: 7.2.0 +appVersion: 7.2.0 sources: - https://github.com/elastic/kibana icon: https://helm.elastic.co/icons/kibana.png diff --git a/kibana/README.md b/kibana/README.md index 2b6eb99ae..be1a8a36a 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -17,7 +17,7 @@ This helm chart is a lightweight way to configure and run our official [Kibana d ``` * Install it ``` - helm install --name kibana elastic/kibana --version 7.1.1 + helm install --name kibana elastic/kibana --version 7.2.0 ``` ## Compatibility @@ -26,14 +26,14 @@ This chart is tested with the latest supported versions. The currently tested ve | 6.x | 7.x | | ----- | ----- | -| 6.8.0 | 7.1.1 | +| 6.8.1 | 7.2.0 | Examples of installing older major versions can be found in the [examples](./examples) directory. -While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.1.1` of Kibana it would look like this: +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.2.0` of Kibana it would look like this: ``` -helm install --name kibana elastic/kibana --version 7.1.1 --set imageTag=7.1.1 +helm install --name kibana elastic/kibana --version 7.2.0 --set imageTag=7.2.0 ``` ## Configuration @@ -46,7 +46,7 @@ helm install --name kibana elastic/kibana --version 7.1.1 --set imageTag=7.1.1 | `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | | `secretMounts` | Allows you easily mount a secret as a file inside the deployment. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | | `image` | The Kibana docker image | `docker.elastic.co/kibana/kibana` | -| `imageTag` | The Kibana docker image tag | `7.1.1` | +| `imageTag` | The Kibana docker image tag | `7.2.0` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 2Gi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | | `protocol` | The protocol that will be used for the readinessProbe. Change this to `https` if you have `server.ssl.enabled: true` set | `http` | diff --git a/kibana/examples/6.x/test/goss.yaml b/kibana/examples/6.x/test/goss.yaml index 2cd3ab0ce..b2a3e5acc 100644 --- a/kibana/examples/6.x/test/goss.yaml +++ b/kibana/examples/6.x/test/goss.yaml @@ -3,7 +3,7 @@ http: status: 200 timeout: 2000 body: - - '"number":"6.8.0"' + - '"number":"6.8.1"' http://localhost:5601/app/kibana: status: 200 diff --git a/kibana/examples/6.x/values.yml b/kibana/examples/6.x/values.yml index 92edf70fa..0f05a541e 100644 --- a/kibana/examples/6.x/values.yml +++ b/kibana/examples/6.x/values.yml @@ -1,4 +1,4 @@ --- -imageTag: 6.8.0 +imageTag: 6.8.1 elasticsearchHosts: "http://six-master:9200" diff --git a/kibana/examples/default/test/goss.yaml b/kibana/examples/default/test/goss.yaml index 5710496ce..cd40d36cd 100644 --- a/kibana/examples/default/test/goss.yaml +++ b/kibana/examples/default/test/goss.yaml @@ -3,7 +3,7 @@ http: status: 200 timeout: 2000 body: - - '"number":"7.1.1"' + - '"number":"7.2.0"' http://localhost:5601/app/kibana: status: 200 diff --git a/kibana/values.yaml b/kibana/values.yaml index f2a47fcf5..e1581bef6 100755 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -21,7 +21,7 @@ secretMounts: [] # path: /usr/share/elasticsearch/config/certs image: "docker.elastic.co/kibana/kibana" -imageTag: "7.1.1" +imageTag: "7.2.0" imagePullPolicy: "IfNotPresent" resources: From b786a50e6b45b99bdf0ba95131ddca7fe4a6f707 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 10:55:26 +0200 Subject: [PATCH 050/115] [elasticsearch] Disable masterTerminationFix by default Closes: #191 This fix which was added for #63 is no longer needed in Elasticsearch > 7.2.0 as this has been fixed upstream. --- elasticsearch/README.md | 1 + elasticsearch/templates/statefulset.yaml | 2 ++ elasticsearch/tests/elasticsearch_test.py | 16 ++++++++++++++++ elasticsearch/values.yaml | 3 +++ 4 files changed, 22 insertions(+) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 1400fd681..aaad572c7 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -104,6 +104,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 --set im | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | | `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Elasticsearch service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | | `schedulerName` | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods) | `nil` | +| `masterTerminationFix` | A workaround needed for Elasticsearch < 7.2.0 to prevent master status being lost during restarts [#63](https://github.com/elastic/helm-charts/issues/63) | `false` | ## Try it out diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index be2d44f8e..298f4fabe 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -222,6 +222,7 @@ spec: {{- if .Values.extraVolumeMounts }} {{ tpl .Values.extraVolumeMounts . | indent 10 }} {{- end }} + {{- if .Values.masterTerminationFix }} {{- if eq .Values.roles.master "true" }} # This sidecar will prevent slow master re-election # https://github.com/elastic/helm-charts/issues/63 @@ -272,3 +273,4 @@ spec: {{ toYaml .Values.extraEnvs | indent 10 }} {{- end }} {{- end }} + {{- end }} diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 9422858a2..c367386dc 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -614,3 +614,19 @@ def test_adding_a_nodePort(): r = helm_template(config) assert r['service'][uname]['spec']['ports'][0]['nodePort'] == 30001 + +def test_master_termination_fixed_enabled(): + config = '' + + r = helm_template(config) + + assert len(r['statefulset'][uname]['spec']['template']['spec']['containers']) == 1 + + config = ''' + masterTerminationFix: true + ''' + + r = helm_template(config) + + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][1] + assert c['name'] == 'elasticsearch-master-graceful-termination-handler' diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 2c9472196..c9646a6cf 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -173,3 +173,6 @@ ingress: nameOverride: "" fullnameOverride: "" + +# https://github.com/elastic/helm-charts/issues/63 +masterTerminationFix: false From 51c1d0540548caafa36e2c21da3a6bcc9a8d5e23 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 11:08:58 +0200 Subject: [PATCH 051/115] Update changelog for 7.2.0 release --- CHANGELOG.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 761242a04..1a72412b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,37 @@ * [#128](https://github.com/elastic/helm-charts/pull/128) - @Crazybus - Add ci jobs for metricbeat * [#127](https://github.com/elastic/helm-charts/pull/127) - @Crazybus - WIP add metricbeat chart +--- +## 7.2.0 - 2019/07/01 + +* 7.2.0 as the default stack version +* Updated the beta status messaging and added proper descriptions to each chart [#158](https://github.com/elastic/helm-charts/pull/158) +* Add GKE 1.13 to automated testing suite [#169](https://github.com/elastic/helm-charts/pull/169) and [#181](https://github.com/elastic/helm-charts/pull/181) + +### Elasticsearch + +* [#123](https://github.com/elastic/helm-charts/pull/123) - @kimxogus - Make the service configurable +* [#141](https://github.com/elastic/helm-charts/pull/141) - @satchpx - Add capability to specify alternate scheduler +* [#161](https://github.com/elastic/helm-charts/pull/161) - @Crazybus - Add configurable nodePort to the service spec +* [#170](https://github.com/elastic/helm-charts/pull/170) - @Crazybus - Update security example docs to match reality +* [#182](https://github.com/elastic/helm-charts/pull/182) - @hxquangnhat - Fix secretName field for secretMounts +* [#186](https://github.com/elastic/helm-charts/pull/186) - @Crazybus - Fix pvc annotations with multiple fields +* [#189](https://github.com/elastic/helm-charts/pull/189) - @gnatpat - Add resources to sidecar container + + +### Kibana + +* [#160](https://github.com/elastic/helm-charts/pull/160) - @Crazybus - Add configurable nodePort to the service spec +* [#168](https://github.com/elastic/helm-charts/pull/168) - @Crazybus - Always set server.host to the docker default +* [#172](https://github.com/elastic/helm-charts/pull/172) - @naseemkullah - Run Kibana as the non-root kibana user (1000) +* [#182](https://github.com/elastic/helm-charts/pull/182) - @hxquangnhat - Fix secretName field for secretMounts +* [#184](https://github.com/elastic/helm-charts/pull/184) - @diegofernandes - Fix wildcard support for ingress + +### Filebeat + +* [#182](https://github.com/elastic/helm-charts/pull/182) - @hxquangnhat - Fix secretName field for secretMounts +* [#188](https://github.com/elastic/helm-charts/pull/188) - @cclauss - Fix octal literal to work in both Python 2 and Python 3 + --- ## 7.1.1 - 2019/06/07 From c07ca7edd9c24849a9bc3bd75792f06dbfb0c6b7 Mon Sep 17 00:00:00 2001 From: cclauss Date: Mon, 1 Jul 2019 11:49:12 +0200 Subject: [PATCH 052/115] raw_input() was removed in Python 3 --- helpers/release.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/helpers/release.py b/helpers/release.py index 264b23ddc..811bde945 100644 --- a/helpers/release.py +++ b/helpers/release.py @@ -5,6 +5,11 @@ import subprocess import yaml +try: + raw_input +except NameError: # Python 3 + raw_input = input + os.chdir(os.path.join(os.path.dirname(__file__), '..')) bucket = 'gs://' + os.environ['GCS_BUCKET'] From d44710742fc87d17550c4bb1d55bbf9eae725f9a Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 11:56:21 +0200 Subject: [PATCH 053/115] [elasticsearch] Fix sidecar container resources test This PR was tested and merged in after the sidecar was disabled by default in https://github.com/elastic/helm-charts/pull/194 --- elasticsearch/tests/elasticsearch_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index d7a246abf..79c681112 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -461,6 +461,7 @@ def test_adding_resources_to_initcontainer(): def test_adding_resources_to_sidecar_container(): config = ''' +masterTerminationFix: true sidecarResources: limits: cpu: "100m" From f8aa4bfaa972ac72abb017acb484ffbf457295d2 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Thu, 9 May 2019 08:58:02 +0200 Subject: [PATCH 054/115] WIP: Add metricbeat chart Right now this is literally the filebeat chart with a s/filebeat/metricbeat run over it and the default metricbeat.yml changed. Since it was already working in this state I'm opening up a PR so that it is easier to see the changes made versus filebeat and already get CI testing it. --- helpers/matrix.yml | 3 + metricbeat/.helmignore | 2 + metricbeat/Chart.yaml | 11 ++ metricbeat/Makefile | 1 + metricbeat/README.md | 111 +++++++++++ metricbeat/examples/default/Makefile | 13 ++ metricbeat/examples/default/test/goss.yaml | 47 +++++ metricbeat/templates/NOTES.txt | 2 + metricbeat/templates/_helpers.tpl | 28 +++ metricbeat/templates/clusterrole.yaml | 21 +++ metricbeat/templates/clusterrolebinding.yaml | 19 ++ metricbeat/templates/configmap.yaml | 17 ++ metricbeat/templates/daemonset.yaml | 136 ++++++++++++++ metricbeat/templates/serviceaccount.yaml | 11 ++ metricbeat/tests/metricbeat_test.py | 184 +++++++++++++++++++ metricbeat/values.yaml | 145 +++++++++++++++ 16 files changed, 751 insertions(+) create mode 100644 metricbeat/.helmignore create mode 100755 metricbeat/Chart.yaml create mode 100644 metricbeat/Makefile create mode 100644 metricbeat/README.md create mode 100644 metricbeat/examples/default/Makefile create mode 100644 metricbeat/examples/default/test/goss.yaml create mode 100755 metricbeat/templates/NOTES.txt create mode 100755 metricbeat/templates/_helpers.tpl create mode 100644 metricbeat/templates/clusterrole.yaml create mode 100644 metricbeat/templates/clusterrolebinding.yaml create mode 100644 metricbeat/templates/configmap.yaml create mode 100644 metricbeat/templates/daemonset.yaml create mode 100644 metricbeat/templates/serviceaccount.yaml create mode 100644 metricbeat/tests/metricbeat_test.py create mode 100755 metricbeat/values.yaml diff --git a/helpers/matrix.yml b/helpers/matrix.yml index fbe55177d..41f9f8d71 100644 --- a/helpers/matrix.yml +++ b/helpers/matrix.yml @@ -2,6 +2,7 @@ CHART: - elasticsearch - kibana - filebeat + - metricbeat ES_SUITE: - default - multi @@ -19,6 +20,8 @@ FILEBEAT_SUITE: - oss - security - 6.x +METRICBEAT_SUITE: + - default KUBERNETES_VERSION: - '1.11' - '1.12' diff --git a/metricbeat/.helmignore b/metricbeat/.helmignore new file mode 100644 index 000000000..e12c0b4b9 --- /dev/null +++ b/metricbeat/.helmignore @@ -0,0 +1,2 @@ +tests/ +.pytest_cache/ diff --git a/metricbeat/Chart.yaml b/metricbeat/Chart.yaml new file mode 100755 index 000000000..980783760 --- /dev/null +++ b/metricbeat/Chart.yaml @@ -0,0 +1,11 @@ +description: Metricbeat +home: https://github.com/elastic/helm-charts +maintainers: +- email: helm-charts@elastic.co + name: Elastic +name: metricbeat +version: 7.0.1-alpha1 +appVersion: 7.0.1 +sources: + - https://github.com/elastic/beats +icon: https://helm.elastic.co/icons/metricbeat.png diff --git a/metricbeat/Makefile b/metricbeat/Makefile new file mode 100644 index 000000000..22218a1f6 --- /dev/null +++ b/metricbeat/Makefile @@ -0,0 +1 @@ +include ../helpers/common.mk diff --git a/metricbeat/README.md b/metricbeat/README.md new file mode 100644 index 000000000..a9f1427c8 --- /dev/null +++ b/metricbeat/README.md @@ -0,0 +1,111 @@ +# Metricbeat Helm Chart + +This functionality is in alpha status and may be changed or removed completely in a future release. Elastic will take a best effort approach to fix any issues, but alpha features are not subject to the support SLA of official GA features. + +This helm chart is a lightweight way to configure and run our official [Metricbeat docker image](https://www.elastic.co/guide/en/beats/metricbeat/current/running-on-docker.html). + +## Requirements + +* Kubernetes >= 1.8 +* [Helm](https://helm.sh/) >= 2.8.0 + +## Installing + +* Add the elastic helm charts repo + ``` + helm repo add elastic https://helm.elastic.co + ``` +* Install it + ``` + helm install --name metricbeat elastic/metricbeat --version 7.0.1-alpha1 + ``` + +## Compatibility + +This chart is tested with the latest supported versions. The currently tested versions are: + +| 5.x | 6.x | 7.x | +| ------ | ----- | ----- | +| 5.6.16 | 6.7.2 | 7.0.1 | + +Examples of installing older major versions can be found in the [examples](./examples) directory. + +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.0.1` of metricbeat it would look like this: + +``` +helm install --name metricbeat elastic/metricbeat --version 7.0.1-alpha1 --set imageTag=7.0.1 +``` + + +## Configuration +| Parameter | Description | Default | +| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `metricbeatConfig` | Allows you to add any config files in `/usr/share/metricbeat` such as `metricbeat.yml`. See [values.yaml](./values.yaml) for an example of the formatting with the default configuration. | see [values.yaml](./values.yaml) | +| `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | +| `extraVolumeMounts` | Any extra volumes mounts to define for the Metricbeat container | `[]` | +| `extraVolumes` | Any extra volumes to define for the pod | `[]` | +| `hostPathRoot` | Fully-qualified [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) that will be used to persist Metricbeat registry data | `/var/lib` | +| `image` | The Metricbeat docker image | `docker.elastic.co/beats/metricbeat` | +| `imageTag` | The Metricbeat docker image tag | `7.0.1` | +| `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | +| `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | +| `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles. | `true` | +| `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Metricbeat pods | `{}` | +| `podSecurityContext` | Configurable [podSecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for Metricbeat pod execution environment | `fsGroup: 1000`
`runAsUser: 0`
`privileged: false` | +| `livenessProbe` | Parameters to pass to [liveness probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) checks for values such as timeouts and thresholds. | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | +| `readinessProbe` | Parameters to pass to [readiness probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) checks for values such as timeouts and thresholds. | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | +| `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the `DaemonSet` | `requests.cpu: 100m`
`requests.memory: 100Mi`
`limits.cpu: 1000m`
`limits.memory: 200Mi` | +| `serviceAccount` | Custom [serviceAccount](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) that Metricbeat will use during execution. By default will use the service account created by this chart. | `""` | +| `secretMounts` | Allows you easily mount a secret as a file inside the `DaemonSet`. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | +| `terminationGracePeriod` | Termination period (in seconds) to wait before killing Metricbeat pod process on pod shutdown | `30` | +| `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | +| `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) for the `DaemonSet`. By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually. | `RollingUpdate` | + +## Examples + +In [examples/](./examples) you will find some example configurations. These examples are used for the automated testing of this helm chart. + +### Default + +* Deploy the [default Elasticsearch helm chart](../elasticsearch/README.md#default) +* Deploy Metricbeat with the default values + ``` + cd examples/default + make + ``` +* You can now setup a port forward for Elasticsearch to observe Metricbeat indices + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + +## Testing + +This chart uses [pytest](https://docs.pytest.org/en/latest/) to test the templating logic. The dependencies for testing can be installed from the [`requirements.txt`](../requirements.txt) in the parent directory. + +``` +pip install -r ../requirements.txt +make pytest +``` + +You can also use `helm template` to look at the YAML being generated + +``` +make template +``` + +It is possible to run all of the tests and linting inside of a docker container + +``` +make test +``` + +## Integration Testing + +Integration tests are run using [goss](https://github.com/aelsabbahy/goss/blob/master/docs/manual.md) which is a serverspec like tool written in golang. See [goss.yaml](examples/default/test/goss.yaml) for an example of what the tests look like. + +To run the goss tests against the default example: +``` +cd examples/default +make goss +``` diff --git a/metricbeat/examples/default/Makefile b/metricbeat/examples/default/Makefile new file mode 100644 index 000000000..a413f2509 --- /dev/null +++ b/metricbeat/examples/default/Makefile @@ -0,0 +1,13 @@ +default: test + +include ../../../helpers/examples.mk + +RELEASE := helm-metricbeat-default + +install: + helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ + +test: install goss + +purge: + helm del --purge $(RELEASE) diff --git a/metricbeat/examples/default/test/goss.yaml b/metricbeat/examples/default/test/goss.yaml new file mode 100644 index 000000000..63ef52ccc --- /dev/null +++ b/metricbeat/examples/default/test/goss.yaml @@ -0,0 +1,47 @@ +port: + tcp:5066: + listening: true + ip: + - '127.0.0.1' + +mount: + /usr/share/metricbeat/data: + exists: true + /run/docker.sock: + exists: true + /var/lib/docker/containers: + exists: true + opts: + - ro + /usr/share/metricbeat/metricbeat.yml: + exists: true + opts: + - ro + +user: + metricbeat: + exists: true + uid: 1000 + gid: 1000 + +http: + http://elasticsearch-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - 'metricbeat-7.0.1' + +file: + /usr/share/metricbeat/metricbeat.yml: + exists: true + contains: + - 'add_kubernetes_metadata' + - 'output.elasticsearch' + - 'elasticsearch-master:9200' + +command: + cd /usr/share/metricbeat && metricbeat test output: + exit-status: 0 + stdout: + - 'elasticsearch: http://elasticsearch-master:9200' + - 'version: 7.0.1' diff --git a/metricbeat/templates/NOTES.txt b/metricbeat/templates/NOTES.txt new file mode 100755 index 000000000..24cdde2e5 --- /dev/null +++ b/metricbeat/templates/NOTES.txt @@ -0,0 +1,2 @@ +1. Watch all containers come up. + $ kubectl get pods --namespace={{ .Release.Namespace }} -l app={{ template "fullname" . }} -w diff --git a/metricbeat/templates/_helpers.tpl b/metricbeat/templates/_helpers.tpl new file mode 100755 index 000000000..769546335 --- /dev/null +++ b/metricbeat/templates/_helpers.tpl @@ -0,0 +1,28 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Use the fullname if the serviceAccount value is not set +*/}} +{{- define "serviceAccount" -}} +{{- if .Values.serviceAccount }} +{{- .Values.serviceAccount -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} diff --git a/metricbeat/templates/clusterrole.yaml b/metricbeat/templates/clusterrole.yaml new file mode 100644 index 000000000..52d7e5e98 --- /dev/null +++ b/metricbeat/templates/clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if .Values.managedServiceAccount }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ template "serviceAccount" . }}-cluster-role + labels: + app: "{{ template "fullname" . }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + verbs: + - get + - list + - watch +{{- end -}} diff --git a/metricbeat/templates/clusterrolebinding.yaml b/metricbeat/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..b901e2387 --- /dev/null +++ b/metricbeat/templates/clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.managedServiceAccount }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ template "serviceAccount" . }}-cluster-role-binding + labels: + app: "{{ template "fullname" . }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} +roleRef: + kind: ClusterRole + name: {{ template "serviceAccount" . }}-cluster-role + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: {{ template "serviceAccount" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/metricbeat/templates/configmap.yaml b/metricbeat/templates/configmap.yaml new file mode 100644 index 000000000..655173b52 --- /dev/null +++ b/metricbeat/templates/configmap.yaml @@ -0,0 +1,17 @@ +{{- if .Values.metricbeatConfig }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "fullname" . }}-config + labels: + app: "{{ template "fullname" . }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} +data: +{{- range $path, $config := .Values.metricbeatConfig }} + {{ $path }}: | +{{ $config | indent 4 -}} +{{- end -}} +{{- end -}} diff --git a/metricbeat/templates/daemonset.yaml b/metricbeat/templates/daemonset.yaml new file mode 100644 index 000000000..a9e2d22a1 --- /dev/null +++ b/metricbeat/templates/daemonset.yaml @@ -0,0 +1,136 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "fullname" . }} + labels: + app: "{{ template "fullname" . }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} +spec: + selector: + matchLabels: + app: "{{ template "fullname" . }}" + release: {{ .Release.Name | quote }} + updateStrategy: + type: {{ .Values.updateStrategy }} + template: + metadata: + annotations: + {{- range $key, $value := .Values.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{/* This forces a restart if the configmap has changed */}} + {{- if .Values.metricbeatConfig }} + configChecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} + {{- end }} + name: "{{ template "fullname" . }}" + labels: + app: "{{ template "fullname" . }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + spec: + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 6 }} + {{- end }} + serviceAccountName: {{ template "serviceAccount" . }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }} + volumes: + {{- range .Values.secretMounts }} + - name: {{ .name }} + secret: + secretName: {{ .name }} + {{- end }} + {{- if .Values.metricbeatConfig }} + - name: metricbeat-config + configMap: + defaultMode: 0600 + name: {{ template "fullname" . }}-config + {{- end }} + - name: data + hostPath: + path: {{ .Values.hostPathRoot }}/{{ template "fullname" . }}-{{ .Release.Namespace }}-data + type: DirectoryOrCreate + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: varrundockersock + hostPath: + path: /var/run/docker.sock + {{- if .Values.extraVolumes }} +{{ tpl .Values.extraVolumes . | indent 6 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + containers: + - name: "metricbeat" + image: "{{ .Values.image }}:{{ .Values.imageTag }}" + imagePullPolicy: "{{ .Values.imagePullPolicy }}" + args: + - "-e" + - "-E" + - "http.enabled=true" + livenessProbe: + exec: + command: + - sh + - -c + - | + #!/usr/bin/env bash -e + curl --fail 127.0.0.1:5066 +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: + exec: + command: + - sh + - -c + - | + #!/usr/bin/env bash -e + metricbeat test output +{{ toYaml .Values.readinessProbe | indent 10 }} + resources: +{{ toYaml .Values.resources | indent 10 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{- if .Values.extraEnvs }} +{{ toYaml .Values.extraEnvs | indent 8 }} +{{- end }} +{{- if .Values.podSecurityContext }} + securityContext: +{{ toYaml .Values.podSecurityContext | indent 10 }} +{{- end }} + volumeMounts: + {{- range .Values.secretMounts }} + - name: {{ .name }} + mountPath: {{ .path }} + {{- if .subPath }} + subPath: {{ .subPath }} + {{- end }} + {{- end }} + {{- range $path, $config := .Values.metricbeatConfig }} + - name: metricbeat-config + mountPath: /usr/share/metricbeat/{{ $path }} + readOnly: true + subPath: {{ $path }} + {{- end }} + - name: data + mountPath: /usr/share/metricbeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + # Necessary when using autodiscovery; avoid mounting it otherwise + # See: https://www.elastic.co/guide/en/beats/metricbeat/master/configuration-autodiscover.html + - name: varrundockersock + mountPath: /var/run/docker.sock + readOnly: true + {{- if .Values.extraVolumeMounts }} +{{ tpl .Values.extraVolumeMounts . | indent 8 }} + {{- end }} diff --git a/metricbeat/templates/serviceaccount.yaml b/metricbeat/templates/serviceaccount.yaml new file mode 100644 index 000000000..70a7a049b --- /dev/null +++ b/metricbeat/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if .Values.managedServiceAccount }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "serviceAccount" . }} + labels: + app: "{{ template "fullname" . }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} +{{- end -}} diff --git a/metricbeat/tests/metricbeat_test.py b/metricbeat/tests/metricbeat_test.py new file mode 100644 index 000000000..eb14e693a --- /dev/null +++ b/metricbeat/tests/metricbeat_test.py @@ -0,0 +1,184 @@ +import os +import sys +sys.path.insert(1, os.path.join(sys.path[0], '../../helpers')) +from helpers import helm_template +import yaml + +project = 'metricbeat' +name = 'release-name-' + project + + +def test_defaults(): + config = ''' + ''' + + r = helm_template(config) + + assert name in r['daemonset'] + + c = r['daemonset'][name]['spec']['template']['spec']['containers'][0] + assert c['name'] == project + assert c['image'].startswith('docker.elastic.co/beats/' + project + ':') + + assert c['env'][0]['name'] == 'POD_NAMESPACE' + assert c['env'][0]['valueFrom']['fieldRef']['fieldPath'] == 'metadata.namespace' + + assert 'curl --fail 127.0.0.1:5066' in c['livenessProbe']['exec']['command'][-1] + + assert 'metricbeat test output' in c['readinessProbe']['exec']['command'][-1] + + # Empty customizable defaults + assert 'imagePullSecrets' not in r['daemonset'][name]['spec']['template']['spec'] + assert 'tolerations' not in r['daemonset'][name]['spec']['template']['spec'] + + assert r['daemonset'][name]['spec']['updateStrategy']['type'] == 'RollingUpdate' + + assert r['daemonset'][name]['spec']['template']['spec']['serviceAccountName'] == name + + volumes = r['daemonset'][name]['spec']['template']['spec']['volumes'] + assert { + 'name': 'data', + 'hostPath': { + 'path': '/var/lib/release-name-metricbeat-default-data', + 'type': 'DirectoryOrCreate' + } + } in volumes + + +def test_adding_envs(): + config = ''' +extraEnvs: +- name: LOG_LEVEL + value: DEBUG +''' + r = helm_template(config) + envs = r['daemonset'][name]['spec']['template']['spec']['containers'][0]['env'] + assert {'name': 'LOG_LEVEL', 'value': 'DEBUG'} in envs + + +def test_adding_image_pull_secrets(): + config = ''' +imagePullSecrets: + - name: test-registry +''' + r = helm_template(config) + assert r['daemonset'][name]['spec']['template']['spec']['imagePullSecrets'][0]['name'] == 'test-registry' + + +def test_adding_tolerations(): + config = ''' +tolerations: +- key: "key1" + operator: "Equal" + value: "value1" + effect: "NoExecute" + tolerationSeconds: 3600 +''' + r = helm_template(config) + assert r['daemonset'][name]['spec']['template']['spec']['tolerations'][0]['key'] == 'key1' + + +def test_override_the_default_update_strategy(): + config = ''' +updateStrategy: OnDelete +''' + + r = helm_template(config) + assert r['daemonset'][name]['spec']['updateStrategy']['type'] == 'OnDelete' + +def test_setting_a_custom_service_account(): + config = ''' +serviceAccount: notdefault +''' + r = helm_template(config) + assert r['daemonset'][name]['spec']['template']['spec']['serviceAccountName'] == 'notdefault' + +def test_self_managing_rbac_resources(): + config = ''' +managedServiceAccount: false +''' + r = helm_template(config) + assert 'serviceaccount' not in r + assert 'clusterrole' not in r + assert 'clusterrolebinding' not in r + +def test_setting_pod_security_context(): + config = ''' +podSecurityContext: + runAsUser: 1001 + fsGroup: 1002 + privileged: false +''' + r = helm_template(config) + c = r['daemonset'][name]['spec']['template']['spec']['containers'][0] + assert c['securityContext']['runAsUser'] == 1001 + assert c['securityContext']['fsGroup'] == 1002 + assert c['securityContext']['privileged'] == False + +def test_adding_in_metricbeat_config(): + config = ''' +metricbeatConfig: + metricbeat.yml: | + key: + nestedkey: value + dot.notation: test + + other-config.yml: | + hello = world +''' + r = helm_template(config) + c = r['configmap'][name + '-config']['data'] + + assert 'metricbeat.yml' in c + assert 'other-config.yml' in c + + assert 'nestedkey: value' in c['metricbeat.yml'] + assert 'dot.notation: test' in c['metricbeat.yml'] + + assert 'hello = world' in c['other-config.yml'] + + d = r['daemonset'][name]['spec']['template']['spec'] + + assert {'configMap': {'name': name + '-config', 'defaultMode': 0600}, 'name': project + '-config'} in d['volumes'] + assert {'mountPath': '/usr/share/metricbeat/metricbeat.yml', 'name': project + '-config', 'subPath': 'metricbeat.yml', 'readOnly': True} in d['containers'][0]['volumeMounts'] + assert {'mountPath': '/usr/share/metricbeat/other-config.yml', 'name': project + '-config', 'subPath': 'other-config.yml', 'readOnly': True} in d['containers'][0]['volumeMounts'] + + assert 'configChecksum' in r['daemonset'][name]['spec']['template']['metadata']['annotations'] + + +def test_adding_a_secret_mount(): + config = ''' +secretMounts: + - name: elastic-certificates + secretName: elastic-certificates + path: /usr/share/metricbeat/config/certs +''' + r = helm_template(config) + s = r['daemonset'][name]['spec']['template']['spec'] + assert s['containers'][0]['volumeMounts'][0] == { + 'mountPath': '/usr/share/metricbeat/config/certs', + 'name': 'elastic-certificates' + } + assert s['volumes'][0] == { + 'name': 'elastic-certificates', + 'secret': { + 'secretName': 'elastic-certificates' + } + } + + +def test_adding_a_extra_volume_with_volume_mount(): + config = ''' +extraVolumes: | + - name: extras + emptyDir: {} +extraVolumeMounts: | + - name: extras + mountPath: /usr/share/extras + readOnly: true +''' + r = helm_template(config) + extraVolume = r['daemonset'][name]['spec']['template']['spec']['volumes'] + assert {'name': 'extras', 'emptyDir': {}} in extraVolume + extraVolumeMounts = r['daemonset'][name]['spec']['template']['spec']['containers'][0]['volumeMounts'] + assert {'name': 'extras', 'mountPath': '/usr/share/extras', 'readOnly': True} in extraVolumeMounts diff --git a/metricbeat/values.yaml b/metricbeat/values.yaml new file mode 100755 index 000000000..145821bcb --- /dev/null +++ b/metricbeat/values.yaml @@ -0,0 +1,145 @@ +--- +# Allows you to add any config files in /usr/share/metricbeat +# such as metricbeat.yml +metricbeatConfig: + metricbeat.yml: | + system: + hostfs: /hostfs + metricbeat.modules: + - module: kubernetes + metricsets: + - container + - node + - pod + - system + - volume + period: 10s + hosts: ["localhost:10255"] + processors: + - add_kubernetes_metadata: + in_cluster: true + - module: kubernetes + enabled: true + metricsets: + - event + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + processes: ['.*'] + process.include_top_n: + by_cpu: 5 + by_memory: 5 + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + output.elasticsearch: + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' + +metricbeatKubeStateMetricsConfig: + metricbeat.yml: | + metricbeat.modules: + - module: kubernetes + enabled: true + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + period: 10s + hosts: ["${KUBE_STATE_METRICS_HOSTS:kube-state-metrics:8080}"] + output.elasticsearch: + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' + +# Extra environment variables to append to the DaemonSet pod spec. +# This will be appended to the current 'env:' key. You can use any of the kubernetes env +# syntax here +extraEnvs: [] +# - name: MY_ENVIRONMENT_VAR +# value: the_value_goes_here + +extraVolumeMounts: [] + # - name: extras + # mountPath: /usr/share/extras + # readOnly: true + +extraVolumes: [] + # - name: extras + # emptyDir: {} + +# Root directory where metricbeat will write data to in order to persist registry data across pod restarts (file position and other metadata). +hostPathRoot: /var/lib + +image: "docker.elastic.co/beats/metricbeat" +imageTag: "7.0.1" +imagePullPolicy: "IfNotPresent" +imagePullSecrets: [] + +livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + +readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + +# Whether this chart should self-manage its service account, role, and associated role binding. +managedServiceAccount: true + +podAnnotations: {} + # iam.amazonaws.com/role: es-cluster + +# Various pod security context settings. Bear in mind that many of these have an impact on metricbeat functioning properly. +# +# - Filesystem group for the metricbeat user. The official elastic docker images always have an id of 1000. +# - User that the container will execute as. Typically necessary to run as root (0) in order to properly collect host container logs. +# - Whether to execute the metricbeat containers as privileged containers. Typically not necessarily unless running within environments such as OpenShift. +podSecurityContext: + fsGroup: 1000 + runAsUser: 0 + privileged: false + +resources: + requests: + cpu: "100m" + memory: "100Mi" + limits: + cpu: "1000m" + memory: "200Mi" + +# Custom service account override that the pod will use +serviceAccount: "" + +# A list of secrets and their paths to mount inside the pod +# This is useful for mounting certificates for security other sensitive values +secretMounts: [] +# - name: metricbeat-certificates +# secretName: metricbeat-certificates +# path: /usr/share/metricbeat/certs + +# How long to wait for metricbeat pods to stop gracefully +terminationGracePeriod: 30 + +tolerations: [] + +updateStrategy: RollingUpdate + +# Override various naming aspects of this chart +# Only edit these if you know what you're doing +nameOverride: "" +fullnameOverride: "" From bc6a0277665aad196acb724ee78c931415cb3bd4 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 11 Jun 2019 10:18:25 +0200 Subject: [PATCH 055/115] Copy in all of the missing fields from the daemonset --- metricbeat/requirements.lock | 6 ++ metricbeat/requirements.yaml | 4 + metricbeat/templates/deployment.yaml | 123 +++++++++++++++++++++++++++ metricbeat/values.yaml | 3 +- 4 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 metricbeat/requirements.lock create mode 100644 metricbeat/requirements.yaml create mode 100644 metricbeat/templates/deployment.yaml diff --git a/metricbeat/requirements.lock b/metricbeat/requirements.lock new file mode 100644 index 000000000..213c4e3a4 --- /dev/null +++ b/metricbeat/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: kube-state-metrics + repository: https://kubernetes-charts.storage.googleapis.com + version: 1.6.0 +digest: sha256:111c5be854f72db1996a198a473a3e69bd50b7c5f046cf03ee4733d62a612874 +generated: 2019-06-11T09:46:07.710748+02:00 diff --git a/metricbeat/requirements.yaml b/metricbeat/requirements.yaml new file mode 100644 index 000000000..37d378f9c --- /dev/null +++ b/metricbeat/requirements.yaml @@ -0,0 +1,4 @@ +dependencies: + - name: 'kube-state-metrics' + version: '1.6.0' + repository: '@stable' diff --git a/metricbeat/templates/deployment.yaml b/metricbeat/templates/deployment.yaml new file mode 100644 index 000000000..1f5cd9fbb --- /dev/null +++ b/metricbeat/templates/deployment.yaml @@ -0,0 +1,123 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: '{{ template "fullname" . }}-metrics' + labels: + app: '{{ template "fullname" . }}-metrics' + chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' + heritage: '{{ .Release.Service }}' + release: '{{ .Release.Name }}' +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + app: '{{ template "fullname" . }}-metrics' + chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' + heritage: '{{ .Release.Service }}' + release: '{{ .Release.Name }}' + template: + metadata: + annotations: + {{- range $key, $value := .Values.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{/* This forces a restart if the configmap has changed */}} + {{- if .Values.metricbeatConfig }} + configChecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} + {{- end }} + labels: + app: '{{ template "fullname" . }}-metrics' + chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' + heritage: '{{ .Release.Service }}' + release: '{{ .Release.Name }}' + spec: + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 6 }} + {{- end }} + serviceAccountName: {{ template "serviceAccount" . }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }} + volumes: + - name: data + mountPath: /usr/share/metricbeat/data + {{- range .Values.secretMounts }} + - name: {{ .name }} + secret: + secretName: {{ .name }} + {{- end }} + {{- if .Values.metricbeatConfig }} + - name: metricbeat-config + configMap: + defaultMode: 0600 + name: {{ template "fullname" . }}-config + {{- end }} + {{- if .Values.extraVolumes }} +{{ tpl .Values.extraVolumes . | indent 6 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + containers: + - name: "metricbeat" + image: "{{ .Values.image }}:{{ .Values.imageTag }}" + imagePullPolicy: "{{ .Values.imagePullPolicy }}" + args: + - "-c" + - "/usr/share/metricbeat/kube-state-metrics-metricbeat.yml" + - "-e" + - "-E" + - "http.enabled=true" + livenessProbe: + exec: + command: + - sh + - -c + - | + #!/usr/bin/env bash -e + curl --fail 127.0.0.1:5066 + readinessProbe: + exec: + command: + - sh + - -c + - | + #!/usr/bin/env bash -e + metricbeat test output +{{ toYaml .Values.readinessProbe | indent 10 }} + resources: +{{ toYaml .Values.resources | indent 10 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBE_STATE_METRICS_HOSTS + value: "$({{ .Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_HOST):$({{ .Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_PORT_HTTP)" +{{- if .Values.extraEnvs }} +{{ toYaml .Values.extraEnvs | indent 8 }} +{{- end }} +{{- if .Values.podSecurityContext }} + securityContext: +{{ toYaml .Values.podSecurityContext | indent 10 }} +{{- end }} + volumeMounts: + - name: data + mountPath: /usr/share/metricbeat/data + {{- range .Values.secretMounts }} + - name: {{ .name }} + mountPath: {{ .path }} + {{- if .subPath }} + subPath: {{ .subPath }} + {{- end }} + {{- end }} + {{- range $path, $config := .Values.metricbeatConfig }} + - name: metricbeat-config + mountPath: /usr/share/metricbeat/{{ $path }} + readOnly: true + subPath: {{ $path }} + {{- end }} + {{- if .Values.extraVolumeMounts }} +{{ tpl .Values.extraVolumeMounts . | indent 8 }} + {{- end }} diff --git a/metricbeat/values.yaml b/metricbeat/values.yaml index 145821bcb..c2593574b 100755 --- a/metricbeat/values.yaml +++ b/metricbeat/values.yaml @@ -46,8 +46,7 @@ metricbeatConfig: output.elasticsearch: hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' -metricbeatKubeStateMetricsConfig: - metricbeat.yml: | + kube-state-metrics-metricbeat.yml: | metricbeat.modules: - module: kubernetes enabled: true From 0d2fc1fd878e3642c5926824535e3339e549b8f3 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Wed, 12 Jun 2019 15:26:22 +0200 Subject: [PATCH 056/115] Update description and beta disclaimer --- metricbeat/Chart.yaml | 2 +- metricbeat/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/Chart.yaml b/metricbeat/Chart.yaml index 980783760..fa505920e 100755 --- a/metricbeat/Chart.yaml +++ b/metricbeat/Chart.yaml @@ -1,4 +1,4 @@ -description: Metricbeat +description: Official Elastic helm chart for Metricbeat home: https://github.com/elastic/helm-charts maintainers: - email: helm-charts@elastic.co diff --git a/metricbeat/README.md b/metricbeat/README.md index a9f1427c8..559d488f3 100644 --- a/metricbeat/README.md +++ b/metricbeat/README.md @@ -1,6 +1,6 @@ # Metricbeat Helm Chart -This functionality is in alpha status and may be changed or removed completely in a future release. Elastic will take a best effort approach to fix any issues, but alpha features are not subject to the support SLA of official GA features. +This functionality is in beta and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features. This helm chart is a lightweight way to configure and run our official [Metricbeat docker image](https://www.elastic.co/guide/en/beats/metricbeat/current/running-on-docker.html). From 4f41c6cf7f211330a848a74ef686bd303187755e Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Wed, 12 Jun 2019 21:28:39 +0200 Subject: [PATCH 057/115] Fix up linting and tests `helm lint` also tries to lint subcharts. So the makefile now adds `charts/` to the .helmignore before running the linting then adds it back when adding the dependency. --- helpers/common.mk | 7 ++++++- helpers/helm-tester/Dockerfile | 4 +++- metricbeat/templates/deployment.yaml | 12 ++++++------ metricbeat/tests/metricbeat_test.py | 9 ++++----- metricbeat/values.yaml | 4 ++++ 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/helpers/common.mk b/helpers/common.mk index 7f38e2d41..3afe2bfd1 100644 --- a/helpers/common.mk +++ b/helpers/common.mk @@ -3,6 +3,7 @@ default: test .ONESHELL: lint: + grep 'charts/' ./.helmignore || echo 'charts/' >> ./.helmignore helm lint --strict ./ template: @@ -15,7 +16,11 @@ build: pytest: pytest -sv --color=yes -test-all: template lint pytest +deps: + sed --in-place '/charts\//d' ./.helmignore + helm dependency update + +test-all: lint deps template pytest test: build docker run --rm -i --user "$$(id -u):$$(id -g)" -v $$(pwd)/../:/app -w /app/$$(basename $$(pwd)) helm-tester make test-all diff --git a/helpers/helm-tester/Dockerfile b/helpers/helm-tester/Dockerfile index b197fcc39..8843743ee 100644 --- a/helpers/helm-tester/Dockerfile +++ b/helpers/helm-tester/Dockerfile @@ -5,4 +5,6 @@ ENV HELM_VERSION=2.14.0 RUN wget https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ mv linux-amd64/helm /usr/local/bin/ && \ - rm -rf linux-amd64 + rm -rf linux-amd64 && \ + HOME=/ helm init --client-only && \ + chmod 777 -R /.helm diff --git a/metricbeat/templates/deployment.yaml b/metricbeat/templates/deployment.yaml index 1f5cd9fbb..27f20ccb1 100644 --- a/metricbeat/templates/deployment.yaml +++ b/metricbeat/templates/deployment.yaml @@ -89,12 +89,12 @@ spec: resources: {{ toYaml .Values.resources | indent 10 }} env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: KUBE_STATE_METRICS_HOSTS - value: "$({{ .Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_HOST):$({{ .Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_PORT_HTTP)" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBE_STATE_METRICS_HOSTS + value: "$({{ .Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_HOST):$({{ .Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_PORT_HTTP)" {{- if .Values.extraEnvs }} {{ toYaml .Values.extraEnvs | indent 8 }} {{- end }} diff --git a/metricbeat/tests/metricbeat_test.py b/metricbeat/tests/metricbeat_test.py index eb14e693a..5bdb21a7b 100644 --- a/metricbeat/tests/metricbeat_test.py +++ b/metricbeat/tests/metricbeat_test.py @@ -7,7 +7,6 @@ project = 'metricbeat' name = 'release-name-' + project - def test_defaults(): config = ''' ''' @@ -39,7 +38,7 @@ def test_defaults(): assert { 'name': 'data', 'hostPath': { - 'path': '/var/lib/release-name-metricbeat-default-data', + 'path': '/var/lib/' + name + '-default-data', 'type': 'DirectoryOrCreate' } } in volumes @@ -98,9 +97,9 @@ def test_self_managing_rbac_resources(): managedServiceAccount: false ''' r = helm_template(config) - assert 'serviceaccount' not in r - assert 'clusterrole' not in r - assert 'clusterrolebinding' not in r + assert name not in r['serviceaccount'] + assert name not in r['clusterrole'] + assert name not in r['clusterrolebinding'] def test_setting_pod_security_context(): config = ''' diff --git a/metricbeat/values.yaml b/metricbeat/values.yaml index c2593574b..256389e74 100755 --- a/metricbeat/values.yaml +++ b/metricbeat/values.yaml @@ -61,6 +61,10 @@ metricbeatConfig: output.elasticsearch: hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' +# Replicas being used for the kube-state-metrics metricbeat deployment + +replicas: 1 + # Extra environment variables to append to the DaemonSet pod spec. # This will be appended to the current 'env:' key. You can use any of the kubernetes env # syntax here From 6b5cd0c8948dfe276605549b111ffbe4ab4db7fb Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Thu, 13 Jun 2019 20:57:00 +0200 Subject: [PATCH 058/115] Update goss testing to support testing multiple containers --- helpers/examples.mk | 8 ++-- metricbeat/examples/default/Makefile | 8 +++- .../examples/default/test/goss-metrics.yaml | 41 +++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 metricbeat/examples/default/test/goss-metrics.yaml diff --git a/helpers/examples.mk b/helpers/examples.mk index 8319c1014..b0ce2e6e3 100644 --- a/helpers/examples.mk +++ b/helpers/examples.mk @@ -1,8 +1,10 @@ GOSS_VERSION := v0.3.6 +GOSS_FILE ?= goss.yaml +GOSS_SELECTOR ?= release=$(RELEASE) goss: - GOSS_CONTAINER=$$(kubectl get pods -l release=$(RELEASE) -o name | awk -F'/' 'NR==1{ print $$NF }') && \ + GOSS_CONTAINER=$$(kubectl get --no-headers=true pods -l $(GOSS_SELECTOR) -o custom-columns=:metadata.name | sed -n 1p ) && \ echo Testing with pod: $$GOSS_CONTAINER && \ - kubectl cp test/*.yaml $$GOSS_CONTAINER:/tmp/goss.yaml && \ - kubectl exec $$GOSS_CONTAINER -- sh -c "cd /tmp/ && curl -s -L https://github.com/aelsabbahy/goss/releases/download/$(GOSS_VERSION)/goss-linux-amd64 -o goss && chmod +rx ./goss && ./goss validate --retry-timeout 30s --sleep 5s --color --format documentation" + kubectl cp test/$(GOSS_FILE) $$GOSS_CONTAINER:/tmp/$(GOSS_FILE) && \ + kubectl exec $$GOSS_CONTAINER -- sh -c "cd /tmp/ && curl -s -L https://github.com/aelsabbahy/goss/releases/download/$(GOSS_VERSION)/goss-linux-amd64 -o goss && chmod +rx ./goss && ./goss --gossfile $(GOSS_FILE) validate --retry-timeout 30s --sleep 5s --color --format documentation" diff --git a/metricbeat/examples/default/Makefile b/metricbeat/examples/default/Makefile index a413f2509..833f62973 100644 --- a/metricbeat/examples/default/Makefile +++ b/metricbeat/examples/default/Makefile @@ -2,12 +2,16 @@ default: test include ../../../helpers/examples.mk -RELEASE := helm-metricbeat-default +RELEASE = helm-metricbeat-default +GOSS_SELECTOR = release=$(RELEASE),app=helm-metricbeat-default-metricbeat install: helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ -test: install goss +test-metrics: + GOSS_FILE=goss-metrics.yaml make goss GOSS_SELECTOR=release=$(RELEASE),app=helm-metricbeat-default-metricbeat-metrics + +test: install goss test-metrics purge: helm del --purge $(RELEASE) diff --git a/metricbeat/examples/default/test/goss-metrics.yaml b/metricbeat/examples/default/test/goss-metrics.yaml new file mode 100644 index 000000000..de7a9cf6a --- /dev/null +++ b/metricbeat/examples/default/test/goss-metrics.yaml @@ -0,0 +1,41 @@ +port: + tcp:5066: + listening: true + ip: + - '127.0.0.1' + +mount: + /usr/share/metricbeat/data: + exists: true + /usr/share/metricbeat/kube-state-metrics-metricbeat.yml: + exists: true + opts: + - ro + +user: + metricbeat: + exists: true + uid: 1000 + gid: 1000 + +http: + http://elasticsearch-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - 'metricbeat-7.0.1' + +file: + /usr/share/metricbeat/metricbeat.yml: + exists: true + contains: + - 'add_kubernetes_metadata' + - 'output.elasticsearch' + - 'elasticsearch-master:9200' + +command: + cd /usr/share/metricbeat && metricbeat test output: + exit-status: 0 + stdout: + - 'elasticsearch: http://elasticsearch-master:9200' + - 'version: 7.0.1' From a31fed0d92d0bae3583e6844ea0116435bcc3740 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 11:35:31 +0200 Subject: [PATCH 059/115] Add tests to make sure that data is flowing in --- metricbeat/examples/default/test/goss-metrics.yaml | 5 +++++ metricbeat/examples/default/test/goss.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/metricbeat/examples/default/test/goss-metrics.yaml b/metricbeat/examples/default/test/goss-metrics.yaml index de7a9cf6a..a9b42f5c7 100644 --- a/metricbeat/examples/default/test/goss-metrics.yaml +++ b/metricbeat/examples/default/test/goss-metrics.yaml @@ -24,6 +24,11 @@ http: timeout: 2000 body: - 'metricbeat-7.0.1' + http://elasticsearch-master:9200/_search?q=metricset.name:state_deployment: + status: 200 + timeout: 2000 + body: + - 'metricbeat-7.0.1' file: /usr/share/metricbeat/metricbeat.yml: diff --git a/metricbeat/examples/default/test/goss.yaml b/metricbeat/examples/default/test/goss.yaml index 63ef52ccc..172dd1434 100644 --- a/metricbeat/examples/default/test/goss.yaml +++ b/metricbeat/examples/default/test/goss.yaml @@ -30,6 +30,11 @@ http: timeout: 2000 body: - 'metricbeat-7.0.1' + http://elasticsearch-master:9200/_search?q=metricset.name:container: + status: 200 + timeout: 2000 + body: + - 'metricbeat-7.0.1' file: /usr/share/metricbeat/metricbeat.yml: From fa00f3d225b136e163df67c747fc6ba40f005d39 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 11:47:27 +0200 Subject: [PATCH 060/115] Add 6.x example --- helpers/matrix.yml | 1 + metricbeat/examples/6.x/Makefile | 17 +++++++ .../examples/6.x/test/goss-metrics.yaml | 45 ++++++++++++++++ metricbeat/examples/6.x/test/goss.yaml | 51 +++++++++++++++++++ metricbeat/examples/6.x/values.yaml | 5 ++ 5 files changed, 119 insertions(+) create mode 100644 metricbeat/examples/6.x/Makefile create mode 100644 metricbeat/examples/6.x/test/goss-metrics.yaml create mode 100644 metricbeat/examples/6.x/test/goss.yaml create mode 100644 metricbeat/examples/6.x/values.yaml diff --git a/helpers/matrix.yml b/helpers/matrix.yml index 41f9f8d71..94419e87b 100644 --- a/helpers/matrix.yml +++ b/helpers/matrix.yml @@ -22,6 +22,7 @@ FILEBEAT_SUITE: - 6.x METRICBEAT_SUITE: - default + - 6.x KUBERNETES_VERSION: - '1.11' - '1.12' diff --git a/metricbeat/examples/6.x/Makefile b/metricbeat/examples/6.x/Makefile new file mode 100644 index 000000000..05a66d2bc --- /dev/null +++ b/metricbeat/examples/6.x/Makefile @@ -0,0 +1,17 @@ +default: test + +include ../../../helpers/examples.mk + +RELEASE := helm-metricbeat-six +GOSS_SELECTOR = release=$(RELEASE),app=helm-metricbeat-six-metricbeat + +install: + helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + +purge: + helm del --purge $(RELEASE) + +test-metrics: + GOSS_FILE=goss-metrics.yaml make goss GOSS_SELECTOR=release=$(RELEASE),app=helm-metricbeat-six-metricbeat-metrics + +test: install goss test-metrics diff --git a/metricbeat/examples/6.x/test/goss-metrics.yaml b/metricbeat/examples/6.x/test/goss-metrics.yaml new file mode 100644 index 000000000..423df13b5 --- /dev/null +++ b/metricbeat/examples/6.x/test/goss-metrics.yaml @@ -0,0 +1,45 @@ +port: + tcp:5066: + listening: true + ip: + - '127.0.0.1' + +mount: + /usr/share/metricbeat/data: + exists: true + /usr/share/metricbeat/kube-state-metrics-metricbeat.yml: + exists: true + opts: + - ro + +user: + metricbeat: + exists: true + uid: 1000 + gid: 1000 + +http: + http://six-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - 'metricbeat-6.7.2' + http://six-master:9200/_search?q=metricset.name:state_deployment: + status: 200 + timeout: 2000 + body: + - 'metricbeat-6.7.2' + +file: + /usr/share/metricbeat/metricbeat.yml: + exists: true + contains: + - 'add_kubernetes_metadata' + - 'output.elasticsearch' + +command: + cd /usr/share/metricbeat && metricbeat test output: + exit-status: 0 + stdout: + - 'elasticsearch: http://six-master:9200' + - 'version: 6.7.2' diff --git a/metricbeat/examples/6.x/test/goss.yaml b/metricbeat/examples/6.x/test/goss.yaml new file mode 100644 index 000000000..4a44473fd --- /dev/null +++ b/metricbeat/examples/6.x/test/goss.yaml @@ -0,0 +1,51 @@ +port: + tcp:5066: + listening: true + ip: + - '127.0.0.1' + +mount: + /usr/share/metricbeat/data: + exists: true + /run/docker.sock: + exists: true + /var/lib/docker/containers: + exists: true + opts: + - ro + /usr/share/metricbeat/metricbeat.yml: + exists: true + opts: + - ro + +user: + metricbeat: + exists: true + uid: 1000 + gid: 1000 + +http: + http://six-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - 'metricbeat-6.7.2' + http://six-master:9200/_search?q=metricset.name:container: + status: 200 + timeout: 2000 + body: + - 'metricbeat-6.7.2' + +file: + /usr/share/metricbeat/metricbeat.yml: + exists: true + contains: + - 'add_kubernetes_metadata' + - 'output.elasticsearch' + +command: + cd /usr/share/metricbeat && metricbeat test output: + exit-status: 0 + stdout: + - 'elasticsearch: http://six-master:9200' + - 'version: 6.7.2' diff --git a/metricbeat/examples/6.x/values.yaml b/metricbeat/examples/6.x/values.yaml new file mode 100644 index 000000000..cc8084b1e --- /dev/null +++ b/metricbeat/examples/6.x/values.yaml @@ -0,0 +1,5 @@ +imageTag: 6.7.2 + +extraEnvs: + - name: ELASTICSEARCH_HOSTS + value: six-master:9200 From 6ac4ffdc545d3395dfbdde1af08161ed1314afb3 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 13:21:57 +0200 Subject: [PATCH 061/115] Add oss example --- helpers/matrix.yml | 1 + metricbeat/examples/oss/Makefile | 17 +++++++ .../examples/oss/test/goss-metrics.yaml | 45 ++++++++++++++++ metricbeat/examples/oss/test/goss.yaml | 51 +++++++++++++++++++ metricbeat/examples/oss/values.yaml | 5 ++ 5 files changed, 119 insertions(+) create mode 100644 metricbeat/examples/oss/Makefile create mode 100644 metricbeat/examples/oss/test/goss-metrics.yaml create mode 100644 metricbeat/examples/oss/test/goss.yaml create mode 100644 metricbeat/examples/oss/values.yaml diff --git a/helpers/matrix.yml b/helpers/matrix.yml index 94419e87b..b59d39a0f 100644 --- a/helpers/matrix.yml +++ b/helpers/matrix.yml @@ -22,6 +22,7 @@ FILEBEAT_SUITE: - 6.x METRICBEAT_SUITE: - default + - oss - 6.x KUBERNETES_VERSION: - '1.11' diff --git a/metricbeat/examples/oss/Makefile b/metricbeat/examples/oss/Makefile new file mode 100644 index 000000000..0e4828ed0 --- /dev/null +++ b/metricbeat/examples/oss/Makefile @@ -0,0 +1,17 @@ +default: test + +include ../../../helpers/examples.mk + +RELEASE := helm-metricbeat-oss +GOSS_SELECTOR = release=$(RELEASE),app=helm-metricbeat-oss-metricbeat + +install: + helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + +purge: + helm del --purge $(RELEASE) + +test-metrics: + GOSS_FILE=goss-metrics.yaml make goss GOSS_SELECTOR=release=$(RELEASE),app=helm-metricbeat-oss-metricbeat-metrics + +test: install goss test-metrics diff --git a/metricbeat/examples/oss/test/goss-metrics.yaml b/metricbeat/examples/oss/test/goss-metrics.yaml new file mode 100644 index 000000000..fa45daf81 --- /dev/null +++ b/metricbeat/examples/oss/test/goss-metrics.yaml @@ -0,0 +1,45 @@ +port: + tcp:5066: + listening: true + ip: + - '127.0.0.1' + +mount: + /usr/share/metricbeat/data: + exists: true + /usr/share/metricbeat/kube-state-metrics-metricbeat.yml: + exists: true + opts: + - ro + +user: + metricbeat: + exists: true + uid: 1000 + gid: 1000 + +http: + http://oss-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - 'metricbeat-7.0.1' + http://oss-master:9200/_search?q=metricset.name:state_deployment: + status: 200 + timeout: 2000 + body: + - 'metricbeat-7.0.1' + +file: + /usr/share/metricbeat/metricbeat.yml: + exists: true + contains: + - 'add_kubernetes_metadata' + - 'output.elasticsearch' + +command: + cd /usr/share/metricbeat && metricbeat test output: + exit-status: 0 + stdout: + - 'elasticsearch: http://oss-master:9200' + - 'version: 7.0.1' diff --git a/metricbeat/examples/oss/test/goss.yaml b/metricbeat/examples/oss/test/goss.yaml new file mode 100644 index 000000000..0598585a8 --- /dev/null +++ b/metricbeat/examples/oss/test/goss.yaml @@ -0,0 +1,51 @@ +port: + tcp:5066: + listening: true + ip: + - '127.0.0.1' + +mount: + /usr/share/metricbeat/data: + exists: true + /run/docker.sock: + exists: true + /var/lib/docker/containers: + exists: true + opts: + - ro + /usr/share/metricbeat/metricbeat.yml: + exists: true + opts: + - ro + +user: + metricbeat: + exists: true + uid: 1000 + gid: 1000 + +http: + http://oss-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - 'metricbeat-7.0.1' + http://oss-master:9200/_search?q=metricset.name:container: + status: 200 + timeout: 2000 + body: + - 'metricbeat-7.0.1' + +file: + /usr/share/metricbeat/metricbeat.yml: + exists: true + contains: + - 'add_kubernetes_metadata' + - 'output.elasticsearch' + +command: + cd /usr/share/metricbeat && metricbeat test output: + exit-status: 0 + stdout: + - 'elasticsearch: http://oss-master:9200' + - 'version: 7.0.1' diff --git a/metricbeat/examples/oss/values.yaml b/metricbeat/examples/oss/values.yaml new file mode 100644 index 000000000..89f2d453c --- /dev/null +++ b/metricbeat/examples/oss/values.yaml @@ -0,0 +1,5 @@ +image: docker.elastic.co/beats/metricbeat-oss + +extraEnvs: + - name: ELASTICSEARCH_HOSTS + value: oss-master:9200 From a71ece63cf2d2c4b4b9672db8ade7e43c3eb64ec Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 13:31:14 +0200 Subject: [PATCH 062/115] Add security example --- helpers/matrix.yml | 1 + metricbeat/examples/security/Makefile | 17 ++++ .../examples/security/test/goss-metrics.yaml | 51 +++++++++++ metricbeat/examples/security/test/goss.yaml | 57 ++++++++++++ metricbeat/examples/security/values.yaml | 87 +++++++++++++++++++ 5 files changed, 213 insertions(+) create mode 100644 metricbeat/examples/security/Makefile create mode 100644 metricbeat/examples/security/test/goss-metrics.yaml create mode 100644 metricbeat/examples/security/test/goss.yaml create mode 100644 metricbeat/examples/security/values.yaml diff --git a/helpers/matrix.yml b/helpers/matrix.yml index b59d39a0f..d8dbf5396 100644 --- a/helpers/matrix.yml +++ b/helpers/matrix.yml @@ -23,6 +23,7 @@ FILEBEAT_SUITE: METRICBEAT_SUITE: - default - oss + - security - 6.x KUBERNETES_VERSION: - '1.11' diff --git a/metricbeat/examples/security/Makefile b/metricbeat/examples/security/Makefile new file mode 100644 index 000000000..3f92e7fe2 --- /dev/null +++ b/metricbeat/examples/security/Makefile @@ -0,0 +1,17 @@ +default: test + +include ../../../helpers/examples.mk + +RELEASE := helm-metricbeat-security +GOSS_SELECTOR = release=$(RELEASE),app=helm-metricbeat-security-metricbeat + +install: + helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + +purge: + helm del --purge $(RELEASE) + +test-metrics: + GOSS_FILE=goss-metrics.yaml make goss GOSS_SELECTOR=release=$(RELEASE),app=helm-metricbeat-security-metricbeat-metrics + +test: install goss test-metrics diff --git a/metricbeat/examples/security/test/goss-metrics.yaml b/metricbeat/examples/security/test/goss-metrics.yaml new file mode 100644 index 000000000..737bae023 --- /dev/null +++ b/metricbeat/examples/security/test/goss-metrics.yaml @@ -0,0 +1,51 @@ +port: + tcp:5066: + listening: true + ip: + - '127.0.0.1' + +mount: + /usr/share/metricbeat/data: + exists: true + /usr/share/metricbeat/kube-state-metrics-metricbeat.yml: + exists: true + opts: + - ro + +user: + metricbeat: + exists: true + uid: 1000 + gid: 1000 + +http: + https://security-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - 'metricbeat-7.0.1' + allow-insecure: true + username: '{{ .Env.ELASTICSEARCH_USERNAME }}' + password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' + https://security-master:9200/_search?q=metricset.name:state_deployment: + status: 200 + timeout: 2000 + body: + - 'metricbeat-7.0.1' + allow-insecure: true + username: '{{ .Env.ELASTICSEARCH_USERNAME }}' + password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' + +file: + /usr/share/metricbeat/metricbeat.yml: + exists: true + contains: + - 'add_kubernetes_metadata' + - 'output.elasticsearch' + +command: + cd /usr/share/metricbeat && metricbeat test output: + exit-status: 0 + stdout: + - 'elasticsearch: https://security-master:9200' + - 'version: 7.0.1' diff --git a/metricbeat/examples/security/test/goss.yaml b/metricbeat/examples/security/test/goss.yaml new file mode 100644 index 000000000..41c920ff9 --- /dev/null +++ b/metricbeat/examples/security/test/goss.yaml @@ -0,0 +1,57 @@ +port: + tcp:5066: + listening: true + ip: + - '127.0.0.1' + +mount: + /usr/share/metricbeat/data: + exists: true + /run/docker.sock: + exists: true + /var/lib/docker/containers: + exists: true + opts: + - ro + /usr/share/metricbeat/metricbeat.yml: + exists: true + opts: + - ro + +user: + metricbeat: + exists: true + uid: 1000 + gid: 1000 + +http: + https://security-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - 'metricbeat-7.0.1' + allow-insecure: true + username: '{{ .Env.ELASTICSEARCH_USERNAME }}' + password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' + https://security-master:9200/_search?q=metricset.name:container: + status: 200 + timeout: 2000 + body: + - 'metricbeat-7.0.1' + allow-insecure: true + username: '{{ .Env.ELASTICSEARCH_USERNAME }}' + password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' + +file: + /usr/share/metricbeat/metricbeat.yml: + exists: true + contains: + - 'add_kubernetes_metadata' + - 'output.elasticsearch' + +command: + cd /usr/share/metricbeat && metricbeat test output: + exit-status: 0 + stdout: + - 'elasticsearch: https://security-master:9200' + - 'version: 7.0.1' diff --git a/metricbeat/examples/security/values.yaml b/metricbeat/examples/security/values.yaml new file mode 100644 index 000000000..5d3f92501 --- /dev/null +++ b/metricbeat/examples/security/values.yaml @@ -0,0 +1,87 @@ +metricbeatConfig: + metricbeat.yml: | + system: + hostfs: /hostfs + metricbeat.modules: + - module: kubernetes + metricsets: + - container + - node + - pod + - system + - volume + period: 10s + hosts: ["localhost:10255"] + processors: + - add_kubernetes_metadata: + in_cluster: true + - module: kubernetes + enabled: true + metricsets: + - event + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + processes: ['.*'] + process.include_top_n: + by_cpu: 5 + by_memory: 5 + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + + output.elasticsearch: + username: '${ELASTICSEARCH_USERNAME}' + password: '${ELASTICSEARCH_PASSWORD}' + protocol: https + hosts: ["security-master:9200"] + ssl.certificate_authorities: + - /usr/share/metricbeat/config/certs/elastic-certificate.pem + + kube-state-metrics-metricbeat.yml: | + metricbeat.modules: + - module: kubernetes + enabled: true + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + period: 10s + hosts: ["${KUBE_STATE_METRICS_HOSTS:kube-state-metrics:8080}"] + output.elasticsearch: + username: '${ELASTICSEARCH_USERNAME}' + password: '${ELASTICSEARCH_PASSWORD}' + protocol: https + hosts: ["security-master:9200"] + ssl.certificate_authorities: + - /usr/share/metricbeat/config/certs/elastic-certificate.pem + +secretMounts: + - name: elastic-certificate-pem + secretName: elastic-certificate-pem + path: /usr/share/metricbeat/config/certs + +extraEnvs: + - name: 'ELASTICSEARCH_USERNAME' + valueFrom: + secretKeyRef: + name: elastic-credentials + key: username + - name: 'ELASTICSEARCH_PASSWORD' + valueFrom: + secretKeyRef: + name: elastic-credentials + key: password From 1e90f5ecf55cd3ecd0c3f0e05d307d316a9c452d Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 13:32:42 +0200 Subject: [PATCH 063/115] Rebase on master and bump to latest version --- metricbeat/Chart.yaml | 4 ++-- metricbeat/README.md | 10 +++++----- metricbeat/examples/6.x/test/goss.yaml | 6 +++--- metricbeat/examples/6.x/values.yaml | 2 +- metricbeat/examples/default/test/goss.yaml | 6 +++--- metricbeat/examples/oss/test/goss.yaml | 6 +++--- metricbeat/examples/security/test/goss.yaml | 6 +++--- metricbeat/values.yaml | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/metricbeat/Chart.yaml b/metricbeat/Chart.yaml index fa505920e..debbbe01c 100755 --- a/metricbeat/Chart.yaml +++ b/metricbeat/Chart.yaml @@ -4,8 +4,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: metricbeat -version: 7.0.1-alpha1 -appVersion: 7.0.1 +version: 7.2.0 +appVersion: 7.2.0 sources: - https://github.com/elastic/beats icon: https://helm.elastic.co/icons/metricbeat.png diff --git a/metricbeat/README.md b/metricbeat/README.md index 559d488f3..4517460d3 100644 --- a/metricbeat/README.md +++ b/metricbeat/README.md @@ -17,7 +17,7 @@ This helm chart is a lightweight way to configure and run our official [Metricbe ``` * Install it ``` - helm install --name metricbeat elastic/metricbeat --version 7.0.1-alpha1 + helm install --name metricbeat elastic/metricbeat --version 7.2.0 ``` ## Compatibility @@ -26,14 +26,14 @@ This chart is tested with the latest supported versions. The currently tested ve | 5.x | 6.x | 7.x | | ------ | ----- | ----- | -| 5.6.16 | 6.7.2 | 7.0.1 | +| 5.6.16 | 6.8.1 | 7.2.0 | Examples of installing older major versions can be found in the [examples](./examples) directory. -While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.0.1` of metricbeat it would look like this: +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.2.0` of metricbeat it would look like this: ``` -helm install --name metricbeat elastic/metricbeat --version 7.0.1-alpha1 --set imageTag=7.0.1 +helm install --name metricbeat elastic/metricbeat --version 7.2.0 --set imageTag=7.2.0 ``` @@ -46,7 +46,7 @@ helm install --name metricbeat elastic/metricbeat --version 7.0.1-alpha1 --set i | `extraVolumes` | Any extra volumes to define for the pod | `[]` | | `hostPathRoot` | Fully-qualified [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) that will be used to persist Metricbeat registry data | `/var/lib` | | `image` | The Metricbeat docker image | `docker.elastic.co/beats/metricbeat` | -| `imageTag` | The Metricbeat docker image tag | `7.0.1` | +| `imageTag` | The Metricbeat docker image tag | `7.2.0` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | | `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles. | `true` | diff --git a/metricbeat/examples/6.x/test/goss.yaml b/metricbeat/examples/6.x/test/goss.yaml index 4a44473fd..ba21aab47 100644 --- a/metricbeat/examples/6.x/test/goss.yaml +++ b/metricbeat/examples/6.x/test/goss.yaml @@ -29,12 +29,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-6.7.2' + - 'metricbeat-6.8.1' http://six-master:9200/_search?q=metricset.name:container: status: 200 timeout: 2000 body: - - 'metricbeat-6.7.2' + - 'metricbeat-6.8.1' file: /usr/share/metricbeat/metricbeat.yml: @@ -48,4 +48,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://six-master:9200' - - 'version: 6.7.2' + - 'version: 6.8.1' diff --git a/metricbeat/examples/6.x/values.yaml b/metricbeat/examples/6.x/values.yaml index cc8084b1e..d0eeea620 100644 --- a/metricbeat/examples/6.x/values.yaml +++ b/metricbeat/examples/6.x/values.yaml @@ -1,4 +1,4 @@ -imageTag: 6.7.2 +imageTag: 6.8.1 extraEnvs: - name: ELASTICSEARCH_HOSTS diff --git a/metricbeat/examples/default/test/goss.yaml b/metricbeat/examples/default/test/goss.yaml index 172dd1434..edc9fdd2f 100644 --- a/metricbeat/examples/default/test/goss.yaml +++ b/metricbeat/examples/default/test/goss.yaml @@ -29,12 +29,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.0.1' + - 'metricbeat-7.2.0' http://elasticsearch-master:9200/_search?q=metricset.name:container: status: 200 timeout: 2000 body: - - 'metricbeat-7.0.1' + - 'metricbeat-7.2.0' file: /usr/share/metricbeat/metricbeat.yml: @@ -49,4 +49,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://elasticsearch-master:9200' - - 'version: 7.0.1' + - 'version: 7.2.0' diff --git a/metricbeat/examples/oss/test/goss.yaml b/metricbeat/examples/oss/test/goss.yaml index 0598585a8..e7788dd04 100644 --- a/metricbeat/examples/oss/test/goss.yaml +++ b/metricbeat/examples/oss/test/goss.yaml @@ -29,12 +29,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.0.1' + - 'metricbeat-7.2.0' http://oss-master:9200/_search?q=metricset.name:container: status: 200 timeout: 2000 body: - - 'metricbeat-7.0.1' + - 'metricbeat-7.2.0' file: /usr/share/metricbeat/metricbeat.yml: @@ -48,4 +48,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://oss-master:9200' - - 'version: 7.0.1' + - 'version: 7.2.0' diff --git a/metricbeat/examples/security/test/goss.yaml b/metricbeat/examples/security/test/goss.yaml index 41c920ff9..05d6d0a87 100644 --- a/metricbeat/examples/security/test/goss.yaml +++ b/metricbeat/examples/security/test/goss.yaml @@ -29,7 +29,7 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.0.1' + - 'metricbeat-7.2.0' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' @@ -37,7 +37,7 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.0.1' + - 'metricbeat-7.2.0' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' @@ -54,4 +54,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: https://security-master:9200' - - 'version: 7.0.1' + - 'version: 7.2.0' diff --git a/metricbeat/values.yaml b/metricbeat/values.yaml index 256389e74..2d9e85a8d 100755 --- a/metricbeat/values.yaml +++ b/metricbeat/values.yaml @@ -85,7 +85,7 @@ extraVolumes: [] hostPathRoot: /var/lib image: "docker.elastic.co/beats/metricbeat" -imageTag: "7.0.1" +imageTag: "7.2.0" imagePullPolicy: "IfNotPresent" imagePullSecrets: [] From 8bc8a8b75ec292c7e5ad51c8c84bb7a0abac47a6 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 13:43:39 +0200 Subject: [PATCH 064/115] Update readme and backport fixes from filebeat chart --- metricbeat/README.md | 25 +++++++++++++------------ metricbeat/templates/daemonset.yaml | 2 +- metricbeat/templates/deployment.yaml | 2 +- metricbeat/tests/metricbeat_test.py | 6 ++---- metricbeat/values.yaml | 1 - 5 files changed, 17 insertions(+), 19 deletions(-) diff --git a/metricbeat/README.md b/metricbeat/README.md index 4517460d3..9c728ad8b 100644 --- a/metricbeat/README.md +++ b/metricbeat/README.md @@ -24,9 +24,9 @@ This helm chart is a lightweight way to configure and run our official [Metricbe This chart is tested with the latest supported versions. The currently tested versions are: -| 5.x | 6.x | 7.x | -| ------ | ----- | ----- | -| 5.6.16 | 6.8.1 | 7.2.0 | +| 6.x | 7.x | +| ----- | ----- | +| 6.8.1 | 7.2.0 | Examples of installing older major versions can be found in the [examples](./examples) directory. @@ -40,26 +40,27 @@ helm install --name metricbeat elastic/metricbeat --version 7.2.0 --set imageTag ## Configuration | Parameter | Description | Default | | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | -| `metricbeatConfig` | Allows you to add any config files in `/usr/share/metricbeat` such as `metricbeat.yml`. See [values.yaml](./values.yaml) for an example of the formatting with the default configuration. | see [values.yaml](./values.yaml) | +| `metricbeatConfig` | Allows you to add any config files in `/usr/share/metricbeat` such as `metricbeat.yml`. See [values.yaml](./values.yaml) for an example of the formatting with the default configuration. | see [values.yaml](./values.yaml) | | `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | -| `extraVolumeMounts` | Any extra volumes mounts to define for the Metricbeat container | `[]` | +| `extraVolumeMounts` | Any extra volumes mounts to define for the Metricbeat container | `[]` | | `extraVolumes` | Any extra volumes to define for the pod | `[]` | -| `hostPathRoot` | Fully-qualified [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) that will be used to persist Metricbeat registry data | `/var/lib` | -| `image` | The Metricbeat docker image | `docker.elastic.co/beats/metricbeat` | -| `imageTag` | The Metricbeat docker image tag | `7.2.0` | +| `hostPathRoot` | Fully-qualified [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) that will be used to persist Metricbeat registry data | `/var/lib` | +| `image` | The Metricbeat docker image | `docker.elastic.co/beats/metricbeat` | +| `imageTag` | The Metricbeat docker image tag | `7.2.0` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | | `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles. | `true` | -| `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Metricbeat pods | `{}` | -| `podSecurityContext` | Configurable [podSecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for Metricbeat pod execution environment | `fsGroup: 1000`
`runAsUser: 0`
`privileged: false` | +| `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Metricbeat pods | `{}` | +| `podSecurityContext` | Configurable [podSecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for Metricbeat pod execution environment | `runAsUser: 0`
`privileged: false` | | `livenessProbe` | Parameters to pass to [liveness probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) checks for values such as timeouts and thresholds. | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | | `readinessProbe` | Parameters to pass to [readiness probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) checks for values such as timeouts and thresholds. | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | | `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the `DaemonSet` | `requests.cpu: 100m`
`requests.memory: 100Mi`
`limits.cpu: 1000m`
`limits.memory: 200Mi` | -| `serviceAccount` | Custom [serviceAccount](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) that Metricbeat will use during execution. By default will use the service account created by this chart. | `""` | +| `serviceAccount` | Custom [serviceAccount](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) that Metricbeat will use during execution. By default will use the service account created by this chart. | `""` | | `secretMounts` | Allows you easily mount a secret as a file inside the `DaemonSet`. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | -| `terminationGracePeriod` | Termination period (in seconds) to wait before killing Metricbeat pod process on pod shutdown | `30` | +| `terminationGracePeriod` | Termination period (in seconds) to wait before killing Metricbeat pod process on pod shutdown | `30` | | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | | `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) for the `DaemonSet`. By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually. | `RollingUpdate` | +| `replicas` | The replica count for the metricbeat deployment talking to kube-state-metrics | `1` | ## Examples diff --git a/metricbeat/templates/daemonset.yaml b/metricbeat/templates/daemonset.yaml index a9e2d22a1..fd1b12f10 100644 --- a/metricbeat/templates/daemonset.yaml +++ b/metricbeat/templates/daemonset.yaml @@ -42,7 +42,7 @@ spec: {{- range .Values.secretMounts }} - name: {{ .name }} secret: - secretName: {{ .name }} + secretName: {{ .secretName }} {{- end }} {{- if .Values.metricbeatConfig }} - name: metricbeat-config diff --git a/metricbeat/templates/deployment.yaml b/metricbeat/templates/deployment.yaml index 27f20ccb1..ce13df754 100644 --- a/metricbeat/templates/deployment.yaml +++ b/metricbeat/templates/deployment.yaml @@ -44,7 +44,7 @@ spec: {{- range .Values.secretMounts }} - name: {{ .name }} secret: - secretName: {{ .name }} + secretName: {{ .secretName }} {{- end }} {{- if .Values.metricbeatConfig }} - name: metricbeat-config diff --git a/metricbeat/tests/metricbeat_test.py b/metricbeat/tests/metricbeat_test.py index 5bdb21a7b..d578fef6b 100644 --- a/metricbeat/tests/metricbeat_test.py +++ b/metricbeat/tests/metricbeat_test.py @@ -105,13 +105,11 @@ def test_setting_pod_security_context(): config = ''' podSecurityContext: runAsUser: 1001 - fsGroup: 1002 privileged: false ''' r = helm_template(config) c = r['daemonset'][name]['spec']['template']['spec']['containers'][0] assert c['securityContext']['runAsUser'] == 1001 - assert c['securityContext']['fsGroup'] == 1002 assert c['securityContext']['privileged'] == False def test_adding_in_metricbeat_config(): @@ -149,7 +147,7 @@ def test_adding_a_secret_mount(): config = ''' secretMounts: - name: elastic-certificates - secretName: elastic-certificates + secretName: elastic-certificates-name path: /usr/share/metricbeat/config/certs ''' r = helm_template(config) @@ -161,7 +159,7 @@ def test_adding_a_secret_mount(): assert s['volumes'][0] == { 'name': 'elastic-certificates', 'secret': { - 'secretName': 'elastic-certificates' + 'secretName': 'elastic-certificates-name' } } diff --git a/metricbeat/values.yaml b/metricbeat/values.yaml index 2d9e85a8d..7cae07ef4 100755 --- a/metricbeat/values.yaml +++ b/metricbeat/values.yaml @@ -113,7 +113,6 @@ podAnnotations: {} # - User that the container will execute as. Typically necessary to run as root (0) in order to properly collect host container logs. # - Whether to execute the metricbeat containers as privileged containers. Typically not necessarily unless running within environments such as OpenShift. podSecurityContext: - fsGroup: 1000 runAsUser: 0 privileged: false From 8bc136528dfb40ed3fd563edf9322708e6873447 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 13:45:16 +0200 Subject: [PATCH 065/115] Update file pattern to include all goss tests --- helpers/bumper.py | 2 +- metricbeat/examples/6.x/test/goss-metrics.yaml | 6 +++--- metricbeat/examples/default/test/goss-metrics.yaml | 6 +++--- metricbeat/examples/oss/test/goss-metrics.yaml | 6 +++--- metricbeat/examples/security/test/goss-metrics.yaml | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/helpers/bumper.py b/helpers/bumper.py index 41a5500c9..1956ff65e 100755 --- a/helpers/bumper.py +++ b/helpers/bumper.py @@ -13,7 +13,7 @@ } file_patterns = [ - '*/examples/*/test/goss.y*ml', + '*/examples/*/test/goss*.y*ml', '*/examples/*/*.y*ml', '*/README.md', '*/values.y*ml', diff --git a/metricbeat/examples/6.x/test/goss-metrics.yaml b/metricbeat/examples/6.x/test/goss-metrics.yaml index 423df13b5..dba51826f 100644 --- a/metricbeat/examples/6.x/test/goss-metrics.yaml +++ b/metricbeat/examples/6.x/test/goss-metrics.yaml @@ -23,12 +23,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-6.7.2' + - 'metricbeat-6.8.1' http://six-master:9200/_search?q=metricset.name:state_deployment: status: 200 timeout: 2000 body: - - 'metricbeat-6.7.2' + - 'metricbeat-6.8.1' file: /usr/share/metricbeat/metricbeat.yml: @@ -42,4 +42,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://six-master:9200' - - 'version: 6.7.2' + - 'version: 6.8.1' diff --git a/metricbeat/examples/default/test/goss-metrics.yaml b/metricbeat/examples/default/test/goss-metrics.yaml index a9b42f5c7..ebef9220f 100644 --- a/metricbeat/examples/default/test/goss-metrics.yaml +++ b/metricbeat/examples/default/test/goss-metrics.yaml @@ -23,12 +23,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.0.1' + - 'metricbeat-7.2.0' http://elasticsearch-master:9200/_search?q=metricset.name:state_deployment: status: 200 timeout: 2000 body: - - 'metricbeat-7.0.1' + - 'metricbeat-7.2.0' file: /usr/share/metricbeat/metricbeat.yml: @@ -43,4 +43,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://elasticsearch-master:9200' - - 'version: 7.0.1' + - 'version: 7.2.0' diff --git a/metricbeat/examples/oss/test/goss-metrics.yaml b/metricbeat/examples/oss/test/goss-metrics.yaml index fa45daf81..f8ec4fa53 100644 --- a/metricbeat/examples/oss/test/goss-metrics.yaml +++ b/metricbeat/examples/oss/test/goss-metrics.yaml @@ -23,12 +23,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.0.1' + - 'metricbeat-7.2.0' http://oss-master:9200/_search?q=metricset.name:state_deployment: status: 200 timeout: 2000 body: - - 'metricbeat-7.0.1' + - 'metricbeat-7.2.0' file: /usr/share/metricbeat/metricbeat.yml: @@ -42,4 +42,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://oss-master:9200' - - 'version: 7.0.1' + - 'version: 7.2.0' diff --git a/metricbeat/examples/security/test/goss-metrics.yaml b/metricbeat/examples/security/test/goss-metrics.yaml index 737bae023..4955c504f 100644 --- a/metricbeat/examples/security/test/goss-metrics.yaml +++ b/metricbeat/examples/security/test/goss-metrics.yaml @@ -23,7 +23,7 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.0.1' + - 'metricbeat-7.2.0' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' @@ -31,7 +31,7 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.0.1' + - 'metricbeat-7.2.0' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' @@ -48,4 +48,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: https://security-master:9200' - - 'version: 7.0.1' + - 'version: 7.2.0' From 39c3c86979fb70a62a2b96ef0e59ba4ef0466aef Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 14:27:05 +0200 Subject: [PATCH 066/115] Always install helm chart dependencies --- helpers/terraform/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/helpers/terraform/Makefile b/helpers/terraform/Makefile index afdf700ca..2a1598cd9 100644 --- a/helpers/terraform/Makefile +++ b/helpers/terraform/Makefile @@ -62,7 +62,9 @@ up: k8s for i in 1 2 3 4 5; do helm init --wait --upgrade && break || sleep 5; done integration: creds - cd ../../$(CHART)/examples/$(SUITE) && \ + cd ../../$(CHART)/ && \ + helm dependency update && \ + cd ./examples/$(SUITE) && \ make build: From 3608b21c6529d157f502f95e5de27c70e9dcd94a Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 16:37:03 +0200 Subject: [PATCH 067/115] Always run helm client init to make sure deps can be added --- helpers/terraform/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/helpers/terraform/Makefile b/helpers/terraform/Makefile index 2a1598cd9..0052a6c2e 100644 --- a/helpers/terraform/Makefile +++ b/helpers/terraform/Makefile @@ -63,6 +63,7 @@ up: k8s integration: creds cd ../../$(CHART)/ && \ + helm init --client-only && \ helm dependency update && \ cd ./examples/$(SUITE) && \ make From f4ff57ed89c8cc85e8cca9c3ee58b76838b505de Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 1 Jul 2019 17:09:27 +0200 Subject: [PATCH 068/115] Remove data mount from the kube-state-metrics metricbeat --- metricbeat/examples/6.x/test/goss-metrics.yaml | 2 -- metricbeat/examples/default/test/goss-metrics.yaml | 2 -- metricbeat/examples/oss/test/goss-metrics.yaml | 2 -- metricbeat/examples/security/test/goss-metrics.yaml | 2 -- metricbeat/templates/deployment.yaml | 4 ---- 5 files changed, 12 deletions(-) diff --git a/metricbeat/examples/6.x/test/goss-metrics.yaml b/metricbeat/examples/6.x/test/goss-metrics.yaml index dba51826f..fd776f6ad 100644 --- a/metricbeat/examples/6.x/test/goss-metrics.yaml +++ b/metricbeat/examples/6.x/test/goss-metrics.yaml @@ -5,8 +5,6 @@ port: - '127.0.0.1' mount: - /usr/share/metricbeat/data: - exists: true /usr/share/metricbeat/kube-state-metrics-metricbeat.yml: exists: true opts: diff --git a/metricbeat/examples/default/test/goss-metrics.yaml b/metricbeat/examples/default/test/goss-metrics.yaml index ebef9220f..ec57e8bee 100644 --- a/metricbeat/examples/default/test/goss-metrics.yaml +++ b/metricbeat/examples/default/test/goss-metrics.yaml @@ -5,8 +5,6 @@ port: - '127.0.0.1' mount: - /usr/share/metricbeat/data: - exists: true /usr/share/metricbeat/kube-state-metrics-metricbeat.yml: exists: true opts: diff --git a/metricbeat/examples/oss/test/goss-metrics.yaml b/metricbeat/examples/oss/test/goss-metrics.yaml index f8ec4fa53..4b115200d 100644 --- a/metricbeat/examples/oss/test/goss-metrics.yaml +++ b/metricbeat/examples/oss/test/goss-metrics.yaml @@ -5,8 +5,6 @@ port: - '127.0.0.1' mount: - /usr/share/metricbeat/data: - exists: true /usr/share/metricbeat/kube-state-metrics-metricbeat.yml: exists: true opts: diff --git a/metricbeat/examples/security/test/goss-metrics.yaml b/metricbeat/examples/security/test/goss-metrics.yaml index 4955c504f..86434f43a 100644 --- a/metricbeat/examples/security/test/goss-metrics.yaml +++ b/metricbeat/examples/security/test/goss-metrics.yaml @@ -5,8 +5,6 @@ port: - '127.0.0.1' mount: - /usr/share/metricbeat/data: - exists: true /usr/share/metricbeat/kube-state-metrics-metricbeat.yml: exists: true opts: diff --git a/metricbeat/templates/deployment.yaml b/metricbeat/templates/deployment.yaml index ce13df754..96cce2ea6 100644 --- a/metricbeat/templates/deployment.yaml +++ b/metricbeat/templates/deployment.yaml @@ -39,8 +39,6 @@ spec: serviceAccountName: {{ template "serviceAccount" . }} terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }} volumes: - - name: data - mountPath: /usr/share/metricbeat/data {{- range .Values.secretMounts }} - name: {{ .name }} secret: @@ -103,8 +101,6 @@ spec: {{ toYaml .Values.podSecurityContext | indent 10 }} {{- end }} volumeMounts: - - name: data - mountPath: /usr/share/metricbeat/data {{- range .Values.secretMounts }} - name: {{ .name }} mountPath: {{ .path }} From 42c88faf488027e98dc1eaf3d4f20f7c458cca07 Mon Sep 17 00:00:00 2001 From: Naseem Date: Mon, 17 Jun 2019 18:07:55 -0400 Subject: [PATCH 069/115] Run as 1000 Signed-off-by: Naseem --- elasticsearch/README.md | 4 +++- elasticsearch/templates/statefulset.yaml | 7 ++++++- elasticsearch/values.yaml | 16 ++++++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 295e4ad64..d771d7e8f 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -94,7 +94,9 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.1.1 --set im | `service.annotations` | Annotations that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` [Annotations](https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws) | `{}` | | `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) for the statefulset. By default Kubernetes will wait for the cluster to be green after upgrading each pod. Setting this to `OnDelete` will allow you to manually delete each pod during upgrades | `RollingUpdate` | | `maxUnavailable` | The [maxUnavailable](https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget) value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group | `1` | -| `fsGroup` | The Group ID (GID) for [securityContext.fsGroup](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) so that the Elasticsearch user can read from the persistent volume | `1000` | +| `fsGroup (DEPRECATED)` | The Group ID (GID) for [securityContext.fsGroup](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) so that the Elasticsearch user can read from the persistent volume | `` | +| `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000` | +| `securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) for the container | `capabilities.drop:[ALL]`
`runAsNonRoot: true`
`runAsUser: 1000` | | `terminationGracePeriod` | The [terminationGracePeriod](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods) in seconds used when trying to stop the pod | `120` | | `sysctlVmMaxMapCount` | Sets the [sysctl vm.max_map_count](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html#vm-max-map-count) needed for Elasticsearch | `262144` | | `readinessProbe` | Configuration fields for the [readinessProbe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 0c5444094..815e68516 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -49,7 +49,10 @@ spec: schedulerName: "{{ .Values.schedulerName }}" {{- end }} securityContext: - fsGroup: {{ .Values.fsGroup }} +{{ toYaml .Values.podSecurityContext | indent 8 }} + {{- if .Values.fsGroup }} + fsGroup: {{ .Values.fsGroup }} # Deprecated value, please use .Values.podSecurityContext.fsGroup + {{- end }} {{- with .Values.tolerations }} tolerations: {{ toYaml . | indent 6 }} @@ -124,6 +127,8 @@ spec: {{- end }} containers: - name: "{{ template "name" . }}" + securityContext: +{{ toYaml .Values.securityContext | indent 10 }} image: "{{ .Values.image }}:{{ .Values.imageTag }}" imagePullPolicy: "{{ .Values.imagePullPolicy }}" readinessProbe: diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 14d28f71a..5f459823b 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -129,8 +129,20 @@ updateStrategy: RollingUpdate # of your pods to be unavailable during maintenance maxUnavailable: 1 - # GroupID for the elasticsearch user. The official elastic docker images always have the id of 1000 -fsGroup: 1000 +podSecurityContext: + fsGroup: 1000 + +# The following value is deprecated, +# please use the above podSecurityContext.fsGroup instead +fsGroup: "" + +securityContext: + capabilities: + drop: + - ALL + # readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 # How long to wait for elasticsearch to stop gracefully terminationGracePeriod: 120 From 7bbad50437c6ec034dc2dedc27ebf7abbdcd431e Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Tue, 2 Jul 2019 11:01:11 +0200 Subject: [PATCH 070/115] Add option to provide custom start/stop hooks Signed-off-by: Tetiana Kravchenko --- elasticsearch/README.md | 2 + elasticsearch/templates/configmap.yaml | 9 ++- elasticsearch/templates/statefulset.yaml | 75 +++++------------------- elasticsearch/values.yaml | 16 +++++ 4 files changed, 41 insertions(+), 61 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index f4df573bf..c65405b56 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -65,6 +65,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 --set im | `minimumMasterNodes` | The value for [discovery.zen.minimum_master_nodes](https://www.elastic.co/guide/en/elasticsearch/reference/6.7/discovery-settings.html#minimum_master_nodes). Should be set to `(master_eligible_nodes / 2) + 1`. Ignored in Elasticsearch versions >= 7. | `2` | | `esMajorVersion` | Used to set major version specific configuration | `7` | | `esConfig` | Allows you to add any config files in `/usr/share/elasticsearch/config/` such as `elasticsearch.yml` and `log4j2.properties`. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | +| `hooks` | Allows you to add lifecycle preStop, postStart hooks. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | | `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | | `extraVolumes` | Additional volumes to be passed to the `tpl` function | | | `extraVolumeMounts` | Additional volumeMounts to be passed to the `tpl` function | | @@ -106,6 +107,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 --set im | `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Elasticsearch service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | | `schedulerName` | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods) | `nil` | | `masterTerminationFix` | A workaround needed for Elasticsearch < 7.2.0 to prevent master status being lost during restarts [#63](https://github.com/elastic/helm-charts/issues/63) | `false` | +| `lifecycle` | Allows you to add lifecycle configuration. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | ## Try it out diff --git a/elasticsearch/templates/configmap.yaml b/elasticsearch/templates/configmap.yaml index fe5f0c3b8..89760ec01 100644 --- a/elasticsearch/templates/configmap.yaml +++ b/elasticsearch/templates/configmap.yaml @@ -1,5 +1,3 @@ -{{- if .Values.esConfig }} ---- apiVersion: v1 kind: ConfigMap metadata: @@ -10,8 +8,15 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" app: "{{ template "uname" . }}" data: +{{- if .Values.esConfig }} {{- range $path, $config := .Values.esConfig }} {{ $path }}: | {{ $config | indent 4 -}} {{- end -}} {{- end -}} +{{- if .Values.hooks }} +{{- range $path, $config := .Values.hooks }} + {{ $path }}: | +{{ $config | indent 4 -}} +{{- end -}} +{{- end }} diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 8f3a7d53f..cc936f171 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -23,7 +23,7 @@ spec: name: {{ template "uname" . }} {{- with .Values.persistence.annotations }} annotations: -{{ toYaml . | indent 8 }} + {{ toYaml . | indent 4 }} {{- end }} spec: {{ toYaml .Values.volumeClaimTemplate | indent 6 }} @@ -96,10 +96,10 @@ spec: {{- range .Values.secretMounts }} - name: {{ .name }} secret: - secretName: {{ .secretName }} + secretName: {{ .name }} {{- end }} - {{- if .Values.esConfig }} - - name: esconfig + {{- if or .Values.esConfig .Values.hooks }} + - name: config configMap: name: {{ template "uname" . }}-config {{- end }} @@ -215,64 +215,21 @@ spec: {{- end }} {{- end }} {{- range $path, $config := .Values.esConfig }} - - name: esconfig + - name: config mountPath: /usr/share/elasticsearch/config/{{ $path }} subPath: {{ $path }} {{- end -}} + {{- if .Values.hooks }} + {{- range $path, $config := .Values.hooks }} + - name: config + mountPath: /{{ $path }} + subPath: {{ $path }} + {{- end -}} + {{- end }} {{- if .Values.extraVolumeMounts }} {{ tpl .Values.extraVolumeMounts . | indent 10 }} {{- end }} - {{- if .Values.masterTerminationFix }} - {{- if eq .Values.roles.master "true" }} - # This sidecar will prevent slow master re-election - # https://github.com/elastic/helm-charts/issues/63 - - name: elasticsearch-master-graceful-termination-handler - image: "{{ .Values.image }}:{{ .Values.imageTag }}" - imagePullPolicy: "{{ .Values.imagePullPolicy }}" - command: - - "sh" - - -c - - | - #!/usr/bin/env bash - set -eo pipefail - - http () { - local path="${1}" - if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then - BASIC_AUTH="-u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" - else - BASIC_AUTH='' - fi - curl -XGET -s -k --fail ${BASIC_AUTH} {{ .Values.protocol }}://{{ template "masterService" . }}:{{ .Values.httpPort }}${path} - } - - cleanup () { - while true ; do - local master="$(http "/_cat/master?h=node" || echo "")" - if [[ $master == "{{ template "masterService" . }}"* && $master != "${NODE_NAME}" ]]; then - echo "This node is not master." - break - fi - echo "This node is still master, waiting gracefully for it to step down" - sleep 1 - done - - exit 0 - } - - trap cleanup SIGTERM - - sleep infinity & - wait $! - resources: -{{ toYaml .Values.sidecarResources | indent 10 }} - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - {{- if .Values.extraEnvs }} -{{ toYaml .Values.extraEnvs | indent 10 }} - {{- end }} - {{- end }} - {{- end }} +{{- if .Values.lifecycle }} + lifecycle: +{{ toYaml .Values.lifecycle | indent 10 }} +{{- end }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 45846eae4..75507647f 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -27,6 +27,14 @@ esConfig: {} # log4j2.properties: | # key = value +# Add custom preStop, postStart lifecycle hooks +hooks: {} + ## (string) Script to execute prior the pod stops. + # preStop: |- + + ## (string) Script to execute after the pod starts. + # postStart: |- + # Extra environment variables to append to this nodeGroup # This will be appended to the current 'env:' key. You can use any of the kubernetes env # syntax here @@ -184,3 +192,11 @@ fullnameOverride: "" # https://github.com/elastic/helm-charts/issues/63 masterTerminationFix: false + +lifecycle: {} + # preStop: + # exec: + # command: ["/bin/bash","/preStop"] + # preStop: + # exec: + # command: ["/bin/bash","/preStop"] From 3aa3071a26ac3f49034dd56b53239a092be59ffd Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Tue, 2 Jul 2019 11:09:26 +0200 Subject: [PATCH 071/115] revert some changes Signed-off-by: Tetiana Kravchenko --- elasticsearch/templates/statefulset.yaml | 53 +++++++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index cc936f171..5ab92ad1d 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -23,7 +23,7 @@ spec: name: {{ template "uname" . }} {{- with .Values.persistence.annotations }} annotations: - {{ toYaml . | indent 4 }} +{{ toYaml . | indent 8 }} {{- end }} spec: {{ toYaml .Values.volumeClaimTemplate | indent 6 }} @@ -96,7 +96,7 @@ spec: {{- range .Values.secretMounts }} - name: {{ .name }} secret: - secretName: {{ .name }} + secretName: {{ .secretName }} {{- end }} {{- if or .Values.esConfig .Values.hooks }} - name: config @@ -229,6 +229,55 @@ spec: {{- if .Values.extraVolumeMounts }} {{ tpl .Values.extraVolumeMounts . | indent 10 }} {{- end }} + {{- if .Values.masterTerminationFix }} + {{- if eq .Values.roles.master "true" }} + # This sidecar will prevent slow master re-election + # https://github.com/elastic/helm-charts/issues/63 + - name: elasticsearch-master-graceful-termination-handler + image: "{{ .Values.image }}:{{ .Values.imageTag }}" + imagePullPolicy: "{{ .Values.imagePullPolicy }}" + command: + - "sh" + - -c + - | + #!/usr/bin/env bash + set -eo pipefail + http () { + local path="${1}" + if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then + BASIC_AUTH="-u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" + else + BASIC_AUTH='' + fi + curl -XGET -s -k --fail ${BASIC_AUTH} {{ .Values.protocol }}://{{ template "masterService" . }}:{{ .Values.httpPort }}${path} + } + cleanup () { + while true ; do + local master="$(http "/_cat/master?h=node" || echo "")" + if [[ $master == "{{ template "masterService" . }}"* && $master != "${NODE_NAME}" ]]; then + echo "This node is not master." + break + fi + echo "This node is still master, waiting gracefully for it to step down" + sleep 1 + done + exit 0 + } + trap cleanup SIGTERM + sleep infinity & + wait $! + resources: +{{ toYaml .Values.sidecarResources | indent 10 }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + {{- if .Values.extraEnvs }} +{{ toYaml .Values.extraEnvs | indent 10 }} + {{- end }} + {{- end }} + {{- end }} {{- if .Values.lifecycle }} lifecycle: {{ toYaml .Values.lifecycle | indent 10 }} From 6ca958260d0a6d8e5416603f8703b19627bf23ac Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Tue, 2 Jul 2019 11:11:30 +0200 Subject: [PATCH 072/115] revert some changes Signed-off-by: Tetiana Kravchenko --- elasticsearch/templates/statefulset.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 5ab92ad1d..b47680ded 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -242,6 +242,7 @@ spec: - | #!/usr/bin/env bash set -eo pipefail + http () { local path="${1}" if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then @@ -251,6 +252,7 @@ spec: fi curl -XGET -s -k --fail ${BASIC_AUTH} {{ .Values.protocol }}://{{ template "masterService" . }}:{{ .Values.httpPort }}${path} } + cleanup () { while true ; do local master="$(http "/_cat/master?h=node" || echo "")" @@ -261,9 +263,12 @@ spec: echo "This node is still master, waiting gracefully for it to step down" sleep 1 done + exit 0 } + trap cleanup SIGTERM + sleep infinity & wait $! resources: From db535e2ae6c6e4d7642ab395b79f65779ed8935f Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Wed, 3 Jul 2019 11:15:15 +0200 Subject: [PATCH 073/115] values: fix duplication, move hooks closer to the lifecycle; tests: add sugested test Signed-off-by: Tetiana Kravchenko --- elasticsearch/tests/elasticsearch_test.py | 17 +++++++++++++++++ elasticsearch/values.yaml | 20 ++++++++++---------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 79c681112..a2b555f98 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -655,3 +655,20 @@ def test_master_termination_fixed_enabled(): c = r['statefulset'][uname]['spec']['template']['spec']['containers'][1] assert c['name'] == 'elasticsearch-master-graceful-termination-handler' + +def test_lifecycle_hooks(): + config = '' + r = helm_template(config) + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] + assert 'lifecycle' not in c + + config = ''' + lifecycle: + preStop: + exec: + command: ["/bin/bash","/preStop"] + ''' + r = helm_template(config) + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] + + assert c['lifecycle']['preStop']['exec']['command'] == ["/bin/bash","/preStop"] diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 75507647f..598a7ed2d 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -27,14 +27,6 @@ esConfig: {} # log4j2.properties: | # key = value -# Add custom preStop, postStart lifecycle hooks -hooks: {} - ## (string) Script to execute prior the pod stops. - # preStop: |- - - ## (string) Script to execute after the pod starts. - # postStart: |- - # Extra environment variables to append to this nodeGroup # This will be appended to the current 'env:' key. You can use any of the kubernetes env # syntax here @@ -197,6 +189,14 @@ lifecycle: {} # preStop: # exec: # command: ["/bin/bash","/preStop"] - # preStop: + # postStart: # exec: - # command: ["/bin/bash","/preStop"] + # command: ["/bin/bash","/postStart"] + +# Add custom preStop, postStart lifecycle hooks +hooks: {} + ## (string) Script to execute prior the pod stops. + # preStop: |- + + ## (string) Script to execute after the pod starts. + # postStart: |- From a5c334e7c143902e08f17d254d13b613fea8d955 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Fri, 5 Jul 2019 10:17:02 +0200 Subject: [PATCH 074/115] revertchanges: remove hooks variable Signed-off-by: Tetiana Kravchenko --- elasticsearch/README.md | 1 - elasticsearch/templates/configmap.yaml | 8 +------- elasticsearch/templates/statefulset.yaml | 13 +++---------- elasticsearch/values.yaml | 6 ------ 4 files changed, 4 insertions(+), 24 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index c65405b56..cecddb86c 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -65,7 +65,6 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 --set im | `minimumMasterNodes` | The value for [discovery.zen.minimum_master_nodes](https://www.elastic.co/guide/en/elasticsearch/reference/6.7/discovery-settings.html#minimum_master_nodes). Should be set to `(master_eligible_nodes / 2) + 1`. Ignored in Elasticsearch versions >= 7. | `2` | | `esMajorVersion` | Used to set major version specific configuration | `7` | | `esConfig` | Allows you to add any config files in `/usr/share/elasticsearch/config/` such as `elasticsearch.yml` and `log4j2.properties`. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | -| `hooks` | Allows you to add lifecycle preStop, postStart hooks. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | | `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | | `extraVolumes` | Additional volumes to be passed to the `tpl` function | | | `extraVolumeMounts` | Additional volumeMounts to be passed to the `tpl` function | | diff --git a/elasticsearch/templates/configmap.yaml b/elasticsearch/templates/configmap.yaml index 89760ec01..78074a804 100644 --- a/elasticsearch/templates/configmap.yaml +++ b/elasticsearch/templates/configmap.yaml @@ -1,3 +1,4 @@ +{{- if .Values.esConfig }} apiVersion: v1 kind: ConfigMap metadata: @@ -8,15 +9,8 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" app: "{{ template "uname" . }}" data: -{{- if .Values.esConfig }} {{- range $path, $config := .Values.esConfig }} {{ $path }}: | {{ $config | indent 4 -}} {{- end -}} {{- end -}} -{{- if .Values.hooks }} -{{- range $path, $config := .Values.hooks }} - {{ $path }}: | -{{ $config | indent 4 -}} -{{- end -}} -{{- end }} diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index b47680ded..1f268e795 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -98,8 +98,8 @@ spec: secret: secretName: {{ .secretName }} {{- end }} - {{- if or .Values.esConfig .Values.hooks }} - - name: config + {{- if .Values.esConfig }} + - name: esconfig configMap: name: {{ template "uname" . }}-config {{- end }} @@ -215,17 +215,10 @@ spec: {{- end }} {{- end }} {{- range $path, $config := .Values.esConfig }} - - name: config + - name: esconfig mountPath: /usr/share/elasticsearch/config/{{ $path }} subPath: {{ $path }} {{- end -}} - {{- if .Values.hooks }} - {{- range $path, $config := .Values.hooks }} - - name: config - mountPath: /{{ $path }} - subPath: {{ $path }} - {{- end -}} - {{- end }} {{- if .Values.extraVolumeMounts }} {{ tpl .Values.extraVolumeMounts . | indent 10 }} {{- end }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 598a7ed2d..4da57116a 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -194,9 +194,3 @@ lifecycle: {} # command: ["/bin/bash","/postStart"] # Add custom preStop, postStart lifecycle hooks -hooks: {} - ## (string) Script to execute prior the pod stops. - # preStop: |- - - ## (string) Script to execute after the pod starts. - # postStart: |- From dd4856535f35f4dc482c117cab95576eb27fe7e9 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Fri, 5 Jul 2019 10:18:22 +0200 Subject: [PATCH 075/115] remove not needed comment Signed-off-by: Tetiana Kravchenko --- elasticsearch/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 4da57116a..f0329da9c 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -192,5 +192,3 @@ lifecycle: {} # postStart: # exec: # command: ["/bin/bash","/postStart"] - -# Add custom preStop, postStart lifecycle hooks From 58142e46b59d435e7463be551fc61722e6b514a9 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Fri, 5 Jul 2019 10:27:48 +0200 Subject: [PATCH 076/115] default values: update example to make it more generic Signed-off-by: Tetiana Kravchenko --- elasticsearch/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index f0329da9c..0b51349be 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -188,7 +188,7 @@ masterTerminationFix: false lifecycle: {} # preStop: # exec: - # command: ["/bin/bash","/preStop"] + # command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"] # postStart: # exec: - # command: ["/bin/bash","/postStart"] + # command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"] From 7d82075080016c063aa46cba709c14ff3baa40a9 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Fri, 5 Jul 2019 11:02:17 +0200 Subject: [PATCH 077/115] [kibana] Make imagePullPolicy actually do something Fixes: #198 This value was documented and configurable but wasn't implemented in the template. --- kibana/templates/deployment.yaml | 1 + kibana/tests/kibana_test.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml index a9f530a62..11ce57e64 100644 --- a/kibana/templates/deployment.yaml +++ b/kibana/templates/deployment.yaml @@ -64,6 +64,7 @@ spec: securityContext: {{ toYaml .Values.securityContext | indent 10 }} image: "{{ .Values.image }}:{{ .Values.imageTag }}" + imagePullPolicy: "{{ .Values.imagePullPolicy }}" env: {{- if .Values.elasticsearchURL }} - name: ELASTICSEARCH_URL diff --git a/kibana/tests/kibana_test.py b/kibana/tests/kibana_test.py index 5ad4a33b1..45f0f1425 100644 --- a/kibana/tests/kibana_test.py +++ b/kibana/tests/kibana_test.py @@ -333,3 +333,18 @@ def test_override_the_serverHost(): c = r['deployment'][name]['spec']['template']['spec']['containers'][0] assert c['env'][1]['name'] == 'SERVER_HOST' assert c['env'][1]['value'] == 'localhost' + +def test_override_imagePullPolicy(): + config = '' + + r = helm_template(config) + c = r['deployment'][name]['spec']['template']['spec']['containers'][0] + assert c['imagePullPolicy'] == 'IfNotPresent' + + config = ''' + imagePullPolicy: Always + ''' + + r = helm_template(config) + c = r['deployment'][name]['spec']['template']['spec']['containers'][0] + assert c['imagePullPolicy'] == 'Always' From 5ccb191fe4a069b37b13b4627dfd5033d82364a5 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Fri, 5 Jul 2019 15:38:51 +0200 Subject: [PATCH 078/115] [elasticsearch] Automatically detect esMajorVersion for default image Closes: #192 --- elasticsearch/README.md | 8 +++--- elasticsearch/examples/6.x/values.yaml | 1 - elasticsearch/templates/_helpers.tpl | 13 ++++++++++ elasticsearch/templates/statefulset.yaml | 6 +++-- elasticsearch/tests/elasticsearch_test.py | 31 +++++++++++++++++++++++ elasticsearch/values.yaml | 2 +- 6 files changed, 53 insertions(+), 8 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index f08c36e85..22ff400e3 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -63,7 +63,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 --set im | `roles` | A hash map with the [specific roles](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) for the node group | `master: true`
`data: true`
`ingest: true` | | `replicas` | Kubernetes replica count for the statefulset (i.e. how many pods) | `3` | | `minimumMasterNodes` | The value for [discovery.zen.minimum_master_nodes](https://www.elastic.co/guide/en/elasticsearch/reference/6.7/discovery-settings.html#minimum_master_nodes). Should be set to `(master_eligible_nodes / 2) + 1`. Ignored in Elasticsearch versions >= 7. | `2` | -| `esMajorVersion` | Used to set major version specific configuration | `7` | +| `esMajorVersion` | Used to set major version specific configuration. If you are using a custom image and not running the default Elasticsearch version you will need to set this to the version you are running (e.g. `esMajorVersion: 6`) | `""` | | `esConfig` | Allows you to add any config files in `/usr/share/elasticsearch/config/` such as `elasticsearch.yml` and `log4j2.properties`. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | | `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | | `extraVolumes` | Additional volumes to be passed to the `tpl` function | | @@ -95,9 +95,9 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 --set im | `service.annotations` | Annotations that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` [Annotations](https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws) | `{}` | | `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) for the statefulset. By default Kubernetes will wait for the cluster to be green after upgrading each pod. Setting this to `OnDelete` will allow you to manually delete each pod during upgrades | `RollingUpdate` | | `maxUnavailable` | The [maxUnavailable](https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget) value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group | `1` | -| `fsGroup (DEPRECATED)` | The Group ID (GID) for [securityContext.fsGroup](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) so that the Elasticsearch user can read from the persistent volume | `` | -| `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000` | -| `securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) for the container | `capabilities.drop:[ALL]`
`runAsNonRoot: true`
`runAsUser: 1000` | +| `fsGroup (DEPRECATED)` | The Group ID (GID) for [securityContext.fsGroup](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) so that the Elasticsearch user can read from the persistent volume | `` | +| `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000` | +| `securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) for the container | `capabilities.drop:[ALL]`
`runAsNonRoot: true`
`runAsUser: 1000` | | `terminationGracePeriod` | The [terminationGracePeriod](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods) in seconds used when trying to stop the pod | `120` | | `sysctlVmMaxMapCount` | Sets the [sysctl vm.max_map_count](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html#vm-max-map-count) needed for Elasticsearch | `262144` | | `readinessProbe` | Configuration fields for the [readinessProbe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | diff --git a/elasticsearch/examples/6.x/values.yaml b/elasticsearch/examples/6.x/values.yaml index d721bdf69..520335482 100644 --- a/elasticsearch/examples/6.x/values.yaml +++ b/elasticsearch/examples/6.x/values.yaml @@ -2,4 +2,3 @@ clusterName: "six" imageTag: "6.8.1" -esMajorVersion: 6 diff --git a/elasticsearch/templates/_helpers.tpl b/elasticsearch/templates/_helpers.tpl index bb50c198f..6ca92a5a3 100755 --- a/elasticsearch/templates/_helpers.tpl +++ b/elasticsearch/templates/_helpers.tpl @@ -34,3 +34,16 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{ $uname }}-{{ $i }}, {{- end -}} {{- end -}} + +{{- define "esMajorVersion" -}} +{{- if .Values.esMajorVersion -}} +{{ .Values.esMajorVersion }} +{{- else -}} +{{- $version := int (index (.Values.imageTag | splitList ".") 0) -}} + {{- if and (contains "docker.elastic.co/elasticsearch/elasticsearch" .Values.image) (not (eq $version 0)) -}} +{{ $version }} + {{- else -}} +7 + {{- end -}} +{{- end -}} +{{- end -}} diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 7c3cd3c59..46ec24dbc 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -8,6 +8,8 @@ metadata: release: {{ .Release.Name | quote }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" app: "{{ template "uname" . }}" + annotations: + esMajorVersion: "{{ include "esMajorVersion" . }}" spec: serviceName: {{ template "uname" . }}-headless selector: @@ -179,7 +181,7 @@ spec: fieldRef: fieldPath: metadata.name {{- if eq .Values.roles.master "true" }} - {{- if ge (int .Values.esMajorVersion) 7 }} + {{- if ge (int (include "esMajorVersion" .)) 7 }} - name: cluster.initial_master_nodes value: "{{ template "endpoints" .Values }}" {{- else }} @@ -187,7 +189,7 @@ spec: value: "{{ .Values.minimumMasterNodes }}" {{- end }} {{- end }} - {{- if lt (int .Values.esMajorVersion) 7 }} + {{- if lt (int (include "esMajorVersion" .)) 7 }} - name: discovery.zen.ping.unicast.hosts value: "{{ template "masterService" . }}-headless" {{- else }} diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index a2b555f98..1dc016738 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -672,3 +672,34 @@ def test_lifecycle_hooks(): c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] assert c['lifecycle']['preStop']['exec']['command'] == ["/bin/bash","/preStop"] + +def test_esMajorVersion_detect_default_version(): + config = '' + + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '7' + +def test_esMajorVersion_default_to_7_if_not_elastic_image(): + config = ''' + image: notElastic + imageTag: 1.0.0 + ''' + + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '7' + +def test_esMajorVersion_default_to_7_if_no_version_is_found(): + config = ''' + imageTag: not_a_number + ''' + + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '7' + +def test_esMajorVersion_set_to_6_based_on_image_tag(): + config = ''' + imageTag: 6.8.1 + ''' + + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '6' diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index e36717129..4c41f87d6 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -16,7 +16,7 @@ roles: replicas: 3 minimumMasterNodes: 2 -esMajorVersion: 7 +esMajorVersion: "" # Allows you to add any config files in /usr/share/elasticsearch/config/ # such as elasticsearch.yml and log4j2.properties From 009d895ccd7869f91be9c73e2cf108915bb4d29c Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 8 Jul 2019 12:02:18 +0200 Subject: [PATCH 079/115] Add test coverage for esMajorVersion for oss and default value --- elasticsearch/tests/elasticsearch_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 1dc016738..60fb7d163 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -703,3 +703,21 @@ def test_esMajorVersion_set_to_6_based_on_image_tag(): r = helm_template(config) assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '6' + +def test_esMajorVersion_always_wins(): + config = ''' + esMajorVersion: 7 + imageTag: 6.0.0 + ''' + + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '7' + +def test_esMajorVersion_parse_image_tag_for_oss_image(): + config = ''' + image: docker.elastic.co/elasticsearch/elasticsearch-oss + imageTag: 6.3.2 + ''' + + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '6' From b554a66d900ed39a80c7d9c4a7b125ffb3fd68ca Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 8 Jul 2019 13:55:45 +0200 Subject: [PATCH 080/115] [meta] Increase the retry timeout for goss The current timeout of 30 seconds is sometimes just not enough time for the metricbeat goss testing. Increasing this to 300 (5 minutes) means we won't be getting any false positives failing the build. --- helpers/examples.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/examples.mk b/helpers/examples.mk index b0ce2e6e3..e820d4165 100644 --- a/helpers/examples.mk +++ b/helpers/examples.mk @@ -6,5 +6,5 @@ goss: GOSS_CONTAINER=$$(kubectl get --no-headers=true pods -l $(GOSS_SELECTOR) -o custom-columns=:metadata.name | sed -n 1p ) && \ echo Testing with pod: $$GOSS_CONTAINER && \ kubectl cp test/$(GOSS_FILE) $$GOSS_CONTAINER:/tmp/$(GOSS_FILE) && \ - kubectl exec $$GOSS_CONTAINER -- sh -c "cd /tmp/ && curl -s -L https://github.com/aelsabbahy/goss/releases/download/$(GOSS_VERSION)/goss-linux-amd64 -o goss && chmod +rx ./goss && ./goss --gossfile $(GOSS_FILE) validate --retry-timeout 30s --sleep 5s --color --format documentation" + kubectl exec $$GOSS_CONTAINER -- sh -c "cd /tmp/ && curl -s -L https://github.com/aelsabbahy/goss/releases/download/$(GOSS_VERSION)/goss-linux-amd64 -o goss && chmod +rx ./goss && ./goss --gossfile $(GOSS_FILE) validate --retry-timeout 300s --sleep 5s --color --format documentation" From dbe09b16fdb5e15dbe791c24508d92b10d488f84 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Fri, 5 Jul 2019 12:24:21 +0200 Subject: [PATCH 081/115] [kibana] Fixup security install docs Fixes: #193 The default makefile target was switched at some point in time to also include installing secrets from our secret service. This obviously doesn't work so good for users. --- kibana/README.md | 2 +- kibana/examples/security/Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kibana/README.md b/kibana/README.md index be1a8a36a..da245979b 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -92,7 +92,7 @@ In [examples/](./examples) you will find some example configurations. These exam * Deploy Kibana with the security example ``` cd examples/security - make + make install ``` * You can now setup a port forward and access Kibana at http://localhost:5601 with the credentials `elastic:changeme` ``` diff --git a/kibana/examples/security/Makefile b/kibana/examples/security/Makefile index 0411eeb29..20d724ab7 100644 --- a/kibana/examples/security/Makefile +++ b/kibana/examples/security/Makefile @@ -3,10 +3,10 @@ include ../../../helpers/examples.mk RELEASE := helm-kibana-security -install: secrets +install: helm upgrade --wait --timeout=600 --install --values ./security.yml $(RELEASE) ../../ ; \ -test: install goss +test: secrets install goss purge: helm del --purge $(RELEASE) From 662171e10bba4b40af30b3b5ae717ab509e24ae0 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 8 Jul 2019 15:18:19 +0200 Subject: [PATCH 082/115] Update Kibana and Elasticsearch security examples to be generated Closes: #27 This makes it possible to run all of the examples without needing to connect to our secret service, or generate the certificates and create the secrets manually. --- elasticsearch/README.md | 23 +++-------------------- elasticsearch/examples/security/Makefile | 15 +++++++++++---- helpers/bumper.py | 1 + helpers/examples.mk | 1 + kibana/README.md | 7 ++++--- kibana/examples/security/Makefile | 10 +--------- kibana/examples/security/security.yml | 7 ++----- kibana/examples/security/test/goss.yaml | 4 ++-- 8 files changed, 25 insertions(+), 43 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index f08c36e85..6cbf43a2a 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -134,29 +134,12 @@ make ### Security -A cluster with security enabled +A cluster with node to node security and https enabled. This example uses autogenerated certificates and password, for a production deployment you want to generate SSL certificates following the [official docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-tls.html#node-certificates). -* Generate SSL certificates following the [official docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-tls.html#node-certificates) -* Create Kubernetes secrets for authentication credentials and certificates. Replace `$YOUR_SECRET_PASSWORD` with your own password. - ``` - kubectl create secret generic elastic-credentials --from-literal=password=$YOUR_SECRET_PASSWORD --from-literal=username=elastic - kubectl create secret generic elastic-certificates --from-file=elastic-certificates.p12 - ``` -* Deploy! +* Generate the certificates and install Elasticsearch ``` cd examples/security - helm upgrade --wait --timeout=600 --install --values ./security.yml elasticsearch ../../ - ``` -* Attach into one of the containers - - ``` - kubectl exec -ti $(kubectl get --no-headers=true pods -l release=elasticsearch -o custom-columns=:metadata.name | head -n 1 ) bash - ``` - -* Test that authentication is now enabled - ``` - curl -k 'https://localhost:9200/' # This one will fail - curl -k -u $ELASTIC_USERNAME:$ELASTIC_PASSWORD https://localhost:9200/ + make ``` ### FAQ diff --git a/elasticsearch/examples/security/Makefile b/elasticsearch/examples/security/Makefile index 881908c07..f3e57f43f 100644 --- a/elasticsearch/examples/security/Makefile +++ b/elasticsearch/examples/security/Makefile @@ -13,10 +13,17 @@ purge: test: secrets install goss secrets: + rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12 || true && \ kubectl delete secrets elastic-credentials elastic-certificates elastic-certificate-pem || true && \ - vault read -field=value secret/devops-ci/helm-charts/elasticsearch/security/certificates | base64 --decode > elastic-certificates.p12 && \ - vault read -field=value secret/devops-ci/helm-charts/elasticsearch/security/certificate-pem | base64 --decode > elastic-certificate.pem && \ - kubectl create secret generic elastic-credentials --from-literal=password=changeme --from-literal=username=elastic && \ + password=$$([ ! -z "$$ELASTIC_PASSWORD" ] && echo $$ELASTIC_PASSWORD || echo $$(docker run --rm docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) /bin/sh -c "< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c20")) && \ + docker run --rm -i -v $$(pwd):/app -w /app \ + --user $$(id -u):$$(id -g) \ + docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) \ + /bin/sh -c " \ + elasticsearch-certutil ca --out /app/elastic-stack-ca.p12 --pass '' && \ + elasticsearch-certutil cert --ca /app/elastic-stack-ca.p12 --pass '' --ca-pass '' --out /app/elastic-certificates.p12" && \ + openssl pkcs12 -nodes -passin pass:'' -in elastic-certificates.p12 -out elastic-certificate.pem && \ kubectl create secret generic elastic-certificates --from-file=elastic-certificates.p12 && \ kubectl create secret generic elastic-certificate-pem --from-file=elastic-certificate.pem && \ - rm -f elastic-certificates.p12 elastic-certificate.pem + kubectl create secret generic elastic-credentials --from-literal=password=$$password --from-literal=username=elastic && \ + rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12 diff --git a/helpers/bumper.py b/helpers/bumper.py index 1956ff65e..976c88e1f 100755 --- a/helpers/bumper.py +++ b/helpers/bumper.py @@ -15,6 +15,7 @@ file_patterns = [ '*/examples/*/test/goss*.y*ml', '*/examples/*/*.y*ml', + 'helpers/examples.mk', '*/README.md', '*/values.y*ml', '*/Chart.y*ml', diff --git a/helpers/examples.mk b/helpers/examples.mk index e820d4165..ca42dde14 100644 --- a/helpers/examples.mk +++ b/helpers/examples.mk @@ -1,6 +1,7 @@ GOSS_VERSION := v0.3.6 GOSS_FILE ?= goss.yaml GOSS_SELECTOR ?= release=$(RELEASE) +STACK_VERSION := 7.2.0 goss: GOSS_CONTAINER=$$(kubectl get --no-headers=true pods -l $(GOSS_SELECTOR) -o custom-columns=:metadata.name | sed -n 1p ) && \ diff --git a/kibana/README.md b/kibana/README.md index da245979b..60427ed0c 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -92,14 +92,15 @@ In [examples/](./examples) you will find some example configurations. These exam * Deploy Kibana with the security example ``` cd examples/security - make install + make ``` -* You can now setup a port forward and access Kibana at http://localhost:5601 with the credentials `elastic:changeme` +* Setup a port forward and access Kibana at https://localhost:5601 ``` + # Get the auto generated password + kubectl get secret elastic-credentials -o jsonpath='{.data.password}' | base64 --decode kubectl port-forward deployment/helm-kibana-default-kibana 5601 ``` - ## Testing This chart uses [pytest](https://docs.pytest.org/en/latest/) to test the templating logic. The dependencies for testing can be installed from the [`requirements.txt`](../requirements.txt) in the parent directory. diff --git a/kibana/examples/security/Makefile b/kibana/examples/security/Makefile index 20d724ab7..2f0b66a1e 100644 --- a/kibana/examples/security/Makefile +++ b/kibana/examples/security/Makefile @@ -6,15 +6,7 @@ RELEASE := helm-kibana-security install: helm upgrade --wait --timeout=600 --install --values ./security.yml $(RELEASE) ../../ ; \ -test: secrets install goss +test: install goss purge: helm del --purge $(RELEASE) - -secrets: - kubectl delete secret kibana-certificates || true - vault read -field=kibana.crt secret/devops-ci/helm-charts/kibana/security/certificates | base64 --decode > kibana.crt - vault read -field=kibana.key secret/devops-ci/helm-charts/kibana/security/certificates | base64 --decode > kibana.key - kubectl create secret generic kibana-certificates --from-file=kibana.crt --from-file=kibana.key && \ - rm -f kibana.crt kibana.key - diff --git a/kibana/examples/security/security.yml b/kibana/examples/security/security.yml index 3517ed8e3..748358651 100644 --- a/kibana/examples/security/security.yml +++ b/kibana/examples/security/security.yml @@ -18,8 +18,8 @@ kibanaConfig: kibana.yml: | server.ssl: enabled: true - key: /usr/share/kibana/config/certs/kibana/kibana.key - certificate: /usr/share/kibana/config/certs/kibana/kibana.crt + key: /usr/share/kibana/config/certs/elastic-certificate.pem + certificate: /usr/share/kibana/config/certs/elastic-certificate.pem xpack.security.encryptionKey: something_at_least_32_characters elasticsearch.ssl: certificateAuthorities: /usr/share/kibana/config/certs/elastic-certificate.pem @@ -31,6 +31,3 @@ secretMounts: - name: elastic-certificate-pem secretName: elastic-certificate-pem path: /usr/share/kibana/config/certs - - name: kibana-certificates - secretName: kibana-certificates - path: /usr/share/kibana/config/certs/kibana diff --git a/kibana/examples/security/test/goss.yaml b/kibana/examples/security/test/goss.yaml index caf26e40f..4060303ec 100644 --- a/kibana/examples/security/test/goss.yaml +++ b/kibana/examples/security/test/goss.yaml @@ -19,8 +19,8 @@ file: contains: - 'server.ssl:' - ' enabled: true' - - ' key: /usr/share/kibana/config/certs/kibana/kibana.key' - - ' certificate: /usr/share/kibana/config/certs/kibana/kibana.crt' + - ' key: /usr/share/kibana/config/certs/elastic-certificate.pem' + - ' certificate: /usr/share/kibana/config/certs/elastic-certificate.pem' - 'xpack.security.encryptionKey: something_at_least_32_characters' - 'elasticsearch.ssl:' - ' certificateAuthorities: /usr/share/kibana/config/certs/elastic-certificate.pem' From 8b4b03bda0f6fbd5d5ec94687e3b0586f060a6d3 Mon Sep 17 00:00:00 2001 From: anthony_dugouchet Date: Mon, 8 Jul 2019 15:37:20 +0200 Subject: [PATCH 083/115] [filebeat] additionals labels --- filebeat/README.md | 1 + filebeat/templates/daemonset.yaml | 3 +++ filebeat/tests/filebeat_test.py | 8 ++++++++ filebeat/values.yaml | 3 +++ 4 files changed, 15 insertions(+) diff --git a/filebeat/README.md b/filebeat/README.md index fe547488c..2870336ce 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -55,6 +55,7 @@ helm install --name filebeat elastic/filebeat --version 7.2.0 --set imageTag=7.2 | `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | | `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles. | `true` | | `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Filebeat pods | `{}` | +| `labels` | Configurable [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) applied to all Filebeat pods | `{}` | | `podSecurityContext` | Configurable [podSecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for Filebeat pod execution environment | `runAsUser: 0`
`privileged: false` | | `livenessProbe` | Parameters to pass to [liveness probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) checks for values such as timeouts and thresholds. | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | | `readinessProbe` | Parameters to pass to [readiness probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) checks for values such as timeouts and thresholds. | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index ddb0fc4eb..69953df30 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -8,6 +8,9 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} + {{- range $key, $value := .Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} spec: selector: matchLabels: diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 1a80e6b22..00517e9c6 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -180,3 +180,11 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {'name': 'extras', 'emptyDir': {}} in extraVolume extraVolumeMounts = r['daemonset'][name]['spec']['template']['spec']['containers'][0]['volumeMounts'] assert {'name': 'extras', 'mountPath': '/usr/share/extras', 'readOnly': True} in extraVolumeMounts + +def test_adding_pod_labels(): + config = ''' +labels: + app.kubernetes.io/name: filebeat +''' + r = helm_template(config) + assert r['daemonset'][name]['metadata']['labels']['app.kubernetes.io/name'] == 'filebeat' diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 9bb996c57..9cf900692 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -53,6 +53,9 @@ readinessProbe: # Whether this chart should self-manage its service account, role, and associated role binding. managedServiceAccount: true +# additionals labels +labels: {} + podAnnotations: {} # iam.amazonaws.com/role: es-cluster From dd84f33c115eeca00a63deff70125a5968a6f693 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 8 Jul 2019 16:20:07 +0200 Subject: [PATCH 084/115] Mount docker socket into testing container --- helpers/terraform/Dockerfile | 8 ++++++++ helpers/terraform/in-docker | 1 + 2 files changed, 9 insertions(+) diff --git a/helpers/terraform/Dockerfile b/helpers/terraform/Dockerfile index 67b701f12..758f0bb8c 100644 --- a/helpers/terraform/Dockerfile +++ b/helpers/terraform/Dockerfile @@ -4,9 +4,11 @@ ENV VAULT_VERSION 0.9.3 ENV TERRAFORM_VERSION=0.11.7 ENV KUBECTL_VERSION=1.14.1 ENV HELM_VERSION=2.14.0 +ENV DOCKER_VERSION=18.09.7 RUN yum -y install \ make \ + openssl \ unzip \ which @@ -36,3 +38,9 @@ RUN curl -O https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION} mv linux-amd64/helm /usr/local/bin/ && \ rm -rf linux-amd64 && \ helm version --client + +RUN curl -O https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz && \ + tar xfv docker* && \ + mv docker/docker /usr/local/bin && \ + rm -rf docker/ && \ + docker diff --git a/helpers/terraform/in-docker b/helpers/terraform/in-docker index 0cce963f4..7b2f6cf1a 100755 --- a/helpers/terraform/in-docker +++ b/helpers/terraform/in-docker @@ -11,5 +11,6 @@ docker run --rm --interactive \ --env HOME=/app \ --volume "${PWD}/../../:/app" \ --user "$(id -u):$(id -g)" \ + -v /var/run/docker.sock:/var/run/docker.sock \ --workdir "/app/helpers/terraform/" \ "helm-charts" "$@" From 4b4aa906b730b9787fabb495071a8b8831b1c61e Mon Sep 17 00:00:00 2001 From: Ciaran Liedeman Date: Sun, 7 Jul 2019 11:51:10 +0200 Subject: [PATCH 085/115] Added kibana pod annotations --- kibana/README.md | 1 + kibana/templates/deployment.yaml | 3 +++ kibana/tests/kibana_test.py | 8 ++++++++ kibana/values.yaml | 3 +++ 4 files changed, 15 insertions(+) diff --git a/kibana/README.md b/kibana/README.md index be1a8a36a..c37e8e53d 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -48,6 +48,7 @@ helm install --name kibana elastic/kibana --version 7.2.0 --set imageTag=7.2.0 | `image` | The Kibana docker image | `docker.elastic.co/kibana/kibana` | | `imageTag` | The Kibana docker image tag | `7.2.0` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | +| `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Kibana pods | `{}` | | `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 2Gi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | | `protocol` | The protocol that will be used for the readinessProbe. Change this to `https` if you have `server.ssl.enabled: true` set | `http` | | `serverHost` | The [`server.host`](https://www.elastic.co/guide/en/kibana/current/settings.html) Kibana setting. This is set explicitly so that the default always matches what comes with the docker image. | `0.0.0.0` | diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml index a9f530a62..827561c28 100644 --- a/kibana/templates/deployment.yaml +++ b/kibana/templates/deployment.yaml @@ -19,6 +19,9 @@ spec: app: kibana release: {{ .Release.Name | quote }} annotations: + {{- range $key, $value := .Values.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} {{/* This forces a restart if the configmap has changed */}} {{- if .Values.kibanaConfig }} configchecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} diff --git a/kibana/tests/kibana_test.py b/kibana/tests/kibana_test.py index 5ad4a33b1..5cf99f568 100644 --- a/kibana/tests/kibana_test.py +++ b/kibana/tests/kibana_test.py @@ -333,3 +333,11 @@ def test_override_the_serverHost(): c = r['deployment'][name]['spec']['template']['spec']['containers'][0] assert c['env'][1]['name'] == 'SERVER_HOST' assert c['env'][1]['value'] == 'localhost' + +def test_adding_pod_annotations(): + config = ''' +podAnnotations: + iam.amazonaws.com/role: es-role +''' + r = helm_template(config) + assert r['deployment'][name]['spec']['template']['metadata']['annotations']['iam.amazonaws.com/role'] == 'es-role' \ No newline at end of file diff --git a/kibana/values.yaml b/kibana/values.yaml index e1581bef6..e34407e23 100755 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -24,6 +24,9 @@ image: "docker.elastic.co/kibana/kibana" imageTag: "7.2.0" imagePullPolicy: "IfNotPresent" +podAnnotations: {} + # iam.amazonaws.com/role: es-cluster + resources: requests: cpu: "100m" From cb718a92ea29c239410e3323343cc81789eada9f Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 8 Jul 2019 17:17:08 +0200 Subject: [PATCH 086/115] Don't mount in the current directory when generating certs This gets really tricky when you are doing docker in docker because the host path of the host doesn't match up properly with the host of the local machine. This also means that we can remove the "run as the current user id" hack to make sure that mounted files were accessible by the hosted user. --- elasticsearch/examples/security/Makefile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/elasticsearch/examples/security/Makefile b/elasticsearch/examples/security/Makefile index f3e57f43f..ca5ce91bf 100644 --- a/elasticsearch/examples/security/Makefile +++ b/elasticsearch/examples/security/Makefile @@ -13,15 +13,17 @@ purge: test: secrets install goss secrets: - rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12 || true && \ - kubectl delete secrets elastic-credentials elastic-certificates elastic-certificate-pem || true && \ + docker rm -f elastic-helm-charts-certs || true + rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12 || true + kubectl delete secrets elastic-credentials elastic-certificates elastic-certificate-pem || true password=$$([ ! -z "$$ELASTIC_PASSWORD" ] && echo $$ELASTIC_PASSWORD || echo $$(docker run --rm docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) /bin/sh -c "< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c20")) && \ - docker run --rm -i -v $$(pwd):/app -w /app \ - --user $$(id -u):$$(id -g) \ + docker run --name elastic-helm-charts-certs -i -w /app \ docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) \ /bin/sh -c " \ elasticsearch-certutil ca --out /app/elastic-stack-ca.p12 --pass '' && \ elasticsearch-certutil cert --ca /app/elastic-stack-ca.p12 --pass '' --ca-pass '' --out /app/elastic-certificates.p12" && \ + docker cp elastic-helm-charts-certs:/app/elastic-certificates.p12 ./ && \ + docker rm -f elastic-helm-charts-certs && \ openssl pkcs12 -nodes -passin pass:'' -in elastic-certificates.p12 -out elastic-certificate.pem && \ kubectl create secret generic elastic-certificates --from-file=elastic-certificates.p12 && \ kubectl create secret generic elastic-certificate-pem --from-file=elastic-certificate.pem && \ From d0046923d2e9b47d12b098ecf006fc11870e8e66 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 8 Jul 2019 21:18:14 +0200 Subject: [PATCH 087/115] Use generated encryption key for Kibana --- elasticsearch/examples/security/Makefile | 4 ++-- kibana/examples/security/Makefile | 7 ++++++- kibana/examples/security/security.yml | 7 ++++++- kibana/examples/security/test/goss.yaml | 2 +- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/elasticsearch/examples/security/Makefile b/elasticsearch/examples/security/Makefile index ca5ce91bf..a2d447a22 100644 --- a/elasticsearch/examples/security/Makefile +++ b/elasticsearch/examples/security/Makefile @@ -8,6 +8,7 @@ install: helm upgrade --wait --timeout=600 --install --values ./security.yml $(RELEASE) ../../ ; \ purge: + kubectl delete secrets elastic-credentials elastic-certificates elastic-certificate-pem || true helm del --purge $(RELEASE) test: secrets install goss @@ -15,13 +16,12 @@ test: secrets install goss secrets: docker rm -f elastic-helm-charts-certs || true rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12 || true - kubectl delete secrets elastic-credentials elastic-certificates elastic-certificate-pem || true password=$$([ ! -z "$$ELASTIC_PASSWORD" ] && echo $$ELASTIC_PASSWORD || echo $$(docker run --rm docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) /bin/sh -c "< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c20")) && \ docker run --name elastic-helm-charts-certs -i -w /app \ docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) \ /bin/sh -c " \ elasticsearch-certutil ca --out /app/elastic-stack-ca.p12 --pass '' && \ - elasticsearch-certutil cert --ca /app/elastic-stack-ca.p12 --pass '' --ca-pass '' --out /app/elastic-certificates.p12" && \ + elasticsearch-certutil cert --name security-master --ca /app/elastic-stack-ca.p12 --pass '' --ca-pass '' --out /app/elastic-certificates.p12" && \ docker cp elastic-helm-charts-certs:/app/elastic-certificates.p12 ./ && \ docker rm -f elastic-helm-charts-certs && \ openssl pkcs12 -nodes -passin pass:'' -in elastic-certificates.p12 -out elastic-certificate.pem && \ diff --git a/kibana/examples/security/Makefile b/kibana/examples/security/Makefile index 2f0b66a1e..a9af51499 100644 --- a/kibana/examples/security/Makefile +++ b/kibana/examples/security/Makefile @@ -6,7 +6,12 @@ RELEASE := helm-kibana-security install: helm upgrade --wait --timeout=600 --install --values ./security.yml $(RELEASE) ../../ ; \ -test: install goss +test: secrets install goss purge: + kubectl delete secret kibana || true helm del --purge $(RELEASE) + +secrets: + encryptionkey=$$(echo $$(docker run --rm docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) /bin/sh -c "< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c50")) && \ + kubectl create secret generic kibana --from-literal=encryptionkey=$$encryptionkey diff --git a/kibana/examples/security/security.yml b/kibana/examples/security/security.yml index 748358651..dfc9d5d13 100644 --- a/kibana/examples/security/security.yml +++ b/kibana/examples/security/security.yml @@ -13,6 +13,11 @@ extraEnvs: secretKeyRef: name: elastic-credentials key: password + - name: 'KIBANA_ENCRYPTION_KEY' + valueFrom: + secretKeyRef: + name: kibana + key: encryptionkey kibanaConfig: kibana.yml: | @@ -20,7 +25,7 @@ kibanaConfig: enabled: true key: /usr/share/kibana/config/certs/elastic-certificate.pem certificate: /usr/share/kibana/config/certs/elastic-certificate.pem - xpack.security.encryptionKey: something_at_least_32_characters + xpack.security.encryptionKey: ${KIBANA_ENCRYPTION_KEY} elasticsearch.ssl: certificateAuthorities: /usr/share/kibana/config/certs/elastic-certificate.pem verificationMode: certificate diff --git a/kibana/examples/security/test/goss.yaml b/kibana/examples/security/test/goss.yaml index 4060303ec..39e9a4852 100644 --- a/kibana/examples/security/test/goss.yaml +++ b/kibana/examples/security/test/goss.yaml @@ -21,7 +21,7 @@ file: - ' enabled: true' - ' key: /usr/share/kibana/config/certs/elastic-certificate.pem' - ' certificate: /usr/share/kibana/config/certs/elastic-certificate.pem' - - 'xpack.security.encryptionKey: something_at_least_32_characters' + - 'xpack.security.encryptionKey:' - 'elasticsearch.ssl:' - ' certificateAuthorities: /usr/share/kibana/config/certs/elastic-certificate.pem' - ' verificationMode: certificate' From 5586ddb97b2c87900aa6df3b95212fae48e4ccd0 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Fri, 5 Jul 2019 09:09:56 +0200 Subject: [PATCH 088/115] [elasticsearch] Adding testing for security context Template tests for the changes from #171 --- elasticsearch/tests/elasticsearch_test.py | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 60fb7d163..1782d6802 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -721,3 +721,50 @@ def test_esMajorVersion_parse_image_tag_for_oss_image(): r = helm_template(config) assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '6' + +def test_set_pod_security_context(): + config = '' + r = helm_template(config) + assert r['statefulset'][uname]['spec']['template']['spec']['securityContext']['fsGroup'] == 1000 + + config = ''' + podSecurityContext: + fsGroup: 1001 + other: test + ''' + + r = helm_template(config) + + assert r['statefulset'][uname]['spec']['template']['spec']['securityContext']['fsGroup'] == 1001 + assert r['statefulset'][uname]['spec']['template']['spec']['securityContext']['other'] == 'test' + +def test_fsGroup_backwards_compatability(): + config = ''' + fsGroup: 1001 + ''' + + r = helm_template(config) + + assert r['statefulset'][uname]['spec']['template']['spec']['securityContext']['fsGroup'] == 1001 + +def test_set_container_security_context(): + config = '' + + r = helm_template(config) + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] + assert c['securityContext']['capabilities']['drop'] == ['ALL'] + assert c['securityContext']['runAsNonRoot'] == True + assert c['securityContext']['runAsUser'] == 1000 + + config = ''' + securityContext: + runAsUser: 1001 + other: test + ''' + + r = helm_template(config) + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] + assert c['securityContext']['capabilities']['drop'] == ['ALL'] + assert c['securityContext']['runAsNonRoot'] == True + assert c['securityContext']['runAsUser'] == 1001 + assert c['securityContext']['other'] == 'test' From 00fe8cd523a079ddf312cb02a39657f2d929a126 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 9 Jul 2019 12:27:04 +0200 Subject: [PATCH 089/115] [meta] Actually retry 5 times The `for i in {1..5}` was working for my shell locally but in CI it was only retrying once. --- helpers/terraform/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/terraform/Makefile b/helpers/terraform/Makefile index 0052a6c2e..583d2f58c 100644 --- a/helpers/terraform/Makefile +++ b/helpers/terraform/Makefile @@ -69,4 +69,4 @@ integration: creds make build: - for i in {1..5}; do docker build -t helm-charts . && break || sleep 15; done + for i in 1 2 3 4 5; do docker build -t helm-charts . && break || sleep 5; done From 790cb8bb365e3abeeb455452024acfc593d1c8ad Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 9 Jul 2019 19:10:40 +0200 Subject: [PATCH 090/115] [kibana] Update healthCheckPath to mention basePath usage Make it clear that this setting needs to be updated if you are using a custom basePath like in #216 --- kibana/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kibana/README.md b/kibana/README.md index c37e8e53d..1022a6475 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -48,14 +48,14 @@ helm install --name kibana elastic/kibana --version 7.2.0 --set imageTag=7.2.0 | `image` | The Kibana docker image | `docker.elastic.co/kibana/kibana` | | `imageTag` | The Kibana docker image tag | `7.2.0` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | -| `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Kibana pods | `{}` | +| `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Kibana pods | `{}` | | `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 2Gi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | | `protocol` | The protocol that will be used for the readinessProbe. Change this to `https` if you have `server.ssl.enabled: true` set | `http` | | `serverHost` | The [`server.host`](https://www.elastic.co/guide/en/kibana/current/settings.html) Kibana setting. This is set explicitly so that the default always matches what comes with the docker image. | `0.0.0.0` | -| `healthCheckPath` | The path used for the readinessProbe to check that Kibana is ready | `/app/kibana` | +| `healthCheckPath` | The path used for the readinessProbe to check that Kibana is ready. If you are setting `server.basePath` you will also need to update this to `/${basePath}/app/kibana` | `/app/kibana` | | `kibanaConfig` | Allows you to add any config files in `/usr/share/kibana/config/` such as `kibana.yml`. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | -| `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000` | -| `securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) for the container | `capabilities.drop:[ALL]`
`runAsNonRoot: true`
`runAsUser: 1000` | +| `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000` | +| `securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) for the container | `capabilities.drop:[ALL]`
`runAsNonRoot: true`
`runAsUser: 1000` | | `serviceAccount` | Allows you to overwrite the "default" [serviceAccount](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) for the pod | `[]` | | `priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `` | | `antiAffinityTopologyKey` | The [anti-affinity topology key](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). By default this will prevent multiple Kibana instances from running on the same Kubernetes node | `kubernetes.io/hostname` | From b1914ada1bf2cfd3ce149b58812b44d80f34767f Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 9 Jul 2019 20:11:42 +0200 Subject: [PATCH 091/115] [meta] Add retries and actual failures for vault auth If you immediately export the result of a command it will never fail even with `set -euo pipefail` --- .ci/jobs/elastic+helm-charts+master+cluster-cleanup.yml | 5 ++++- .ci/jobs/elastic+helm-charts+master+cluster-creation.yml | 5 ++++- .../elastic+helm-charts+master+integration-elasticsearch.yml | 5 ++++- .ci/jobs/elastic+helm-charts+master+integration-filebeat.yml | 5 ++++- .ci/jobs/elastic+helm-charts+master+integration-kibana.yml | 5 ++++- .../elastic+helm-charts+master+integration-metricbeat.yml | 5 ++++- .../elastic+helm-charts+pull-request+cluster-cleanup.yml | 5 ++++- .../elastic+helm-charts+pull-request+cluster-creation.yml | 5 ++++- ...ic+helm-charts+pull-request+integration-elasticsearch.yml | 5 ++++- ...elastic+helm-charts+pull-request+integration-filebeat.yml | 5 ++++- .../elastic+helm-charts+pull-request+integration-kibana.yml | 5 ++++- ...astic+helm-charts+pull-request+integration-metricbeat.yml | 5 ++++- 12 files changed, 48 insertions(+), 12 deletions(-) diff --git a/.ci/jobs/elastic+helm-charts+master+cluster-cleanup.yml b/.ci/jobs/elastic+helm-charts+master+cluster-cleanup.yml index 1467a2d44..37862f81a 100644 --- a/.ci/jobs/elastic+helm-charts+master+cluster-cleanup.yml +++ b/.ci/jobs/elastic+helm-charts+master+cluster-cleanup.yml @@ -21,8 +21,11 @@ #!/usr/local/bin/runbld set -euo pipefail + source /usr/local/bin/bash_standard_lib.sh + set +x - export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x diff --git a/.ci/jobs/elastic+helm-charts+master+cluster-creation.yml b/.ci/jobs/elastic+helm-charts+master+cluster-creation.yml index 0a1bd9dab..c1698c76d 100644 --- a/.ci/jobs/elastic+helm-charts+master+cluster-creation.yml +++ b/.ci/jobs/elastic+helm-charts+master+cluster-creation.yml @@ -21,8 +21,11 @@ #!/usr/local/bin/runbld set -euo pipefail + source /usr/local/bin/bash_standard_lib.sh + set +x - export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x diff --git a/.ci/jobs/elastic+helm-charts+master+integration-elasticsearch.yml b/.ci/jobs/elastic+helm-charts+master+integration-elasticsearch.yml index bf3f68450..5b5ccc95c 100644 --- a/.ci/jobs/elastic+helm-charts+master+integration-elasticsearch.yml +++ b/.ci/jobs/elastic+helm-charts+master+integration-elasticsearch.yml @@ -25,8 +25,11 @@ #!/usr/local/bin/runbld set -euo pipefail + source /usr/local/bin/bash_standard_lib.sh + set +x - export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x diff --git a/.ci/jobs/elastic+helm-charts+master+integration-filebeat.yml b/.ci/jobs/elastic+helm-charts+master+integration-filebeat.yml index 0e45fd1cc..661d5e993 100644 --- a/.ci/jobs/elastic+helm-charts+master+integration-filebeat.yml +++ b/.ci/jobs/elastic+helm-charts+master+integration-filebeat.yml @@ -25,8 +25,11 @@ #!/usr/local/bin/runbld set -euo pipefail + source /usr/local/bin/bash_standard_lib.sh + set +x - export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x diff --git a/.ci/jobs/elastic+helm-charts+master+integration-kibana.yml b/.ci/jobs/elastic+helm-charts+master+integration-kibana.yml index 73ac04282..d689e9143 100644 --- a/.ci/jobs/elastic+helm-charts+master+integration-kibana.yml +++ b/.ci/jobs/elastic+helm-charts+master+integration-kibana.yml @@ -25,8 +25,11 @@ #!/usr/local/bin/runbld set -euo pipefail + source /usr/local/bin/bash_standard_lib.sh + set +x - export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x diff --git a/.ci/jobs/elastic+helm-charts+master+integration-metricbeat.yml b/.ci/jobs/elastic+helm-charts+master+integration-metricbeat.yml index 6e3a4663a..480b700e2 100644 --- a/.ci/jobs/elastic+helm-charts+master+integration-metricbeat.yml +++ b/.ci/jobs/elastic+helm-charts+master+integration-metricbeat.yml @@ -25,8 +25,11 @@ #!/usr/local/bin/runbld set -euo pipefail + source /usr/local/bin/bash_standard_lib.sh + set +x - export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x diff --git a/.ci/jobs/elastic+helm-charts+pull-request+cluster-cleanup.yml b/.ci/jobs/elastic+helm-charts+pull-request+cluster-cleanup.yml index 39521ca39..cbd0d55eb 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+cluster-cleanup.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+cluster-cleanup.yml @@ -21,8 +21,11 @@ #!/usr/local/bin/runbld set -euo pipefail + source /usr/local/bin/bash_standard_lib.sh + set +x - export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x diff --git a/.ci/jobs/elastic+helm-charts+pull-request+cluster-creation.yml b/.ci/jobs/elastic+helm-charts+pull-request+cluster-creation.yml index ad0c27d8b..d1b491080 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+cluster-creation.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+cluster-creation.yml @@ -21,8 +21,11 @@ #!/usr/local/bin/runbld set -euo pipefail + source /usr/local/bin/bash_standard_lib.sh + set +x - export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x diff --git a/.ci/jobs/elastic+helm-charts+pull-request+integration-elasticsearch.yml b/.ci/jobs/elastic+helm-charts+pull-request+integration-elasticsearch.yml index 438bc82db..a476a1d85 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+integration-elasticsearch.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+integration-elasticsearch.yml @@ -25,8 +25,11 @@ #!/usr/local/bin/runbld set -euo pipefail + source /usr/local/bin/bash_standard_lib.sh + set +x - export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x diff --git a/.ci/jobs/elastic+helm-charts+pull-request+integration-filebeat.yml b/.ci/jobs/elastic+helm-charts+pull-request+integration-filebeat.yml index fe86038a3..3a1621fbb 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+integration-filebeat.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+integration-filebeat.yml @@ -25,8 +25,11 @@ #!/usr/local/bin/runbld set -euo pipefail + source /usr/local/bin/bash_standard_lib.sh + set +x - export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x diff --git a/.ci/jobs/elastic+helm-charts+pull-request+integration-kibana.yml b/.ci/jobs/elastic+helm-charts+pull-request+integration-kibana.yml index f60f3ac29..ed2e6ec4a 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+integration-kibana.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+integration-kibana.yml @@ -25,8 +25,11 @@ #!/usr/local/bin/runbld set -euo pipefail + source /usr/local/bin/bash_standard_lib.sh + set +x - export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x diff --git a/.ci/jobs/elastic+helm-charts+pull-request+integration-metricbeat.yml b/.ci/jobs/elastic+helm-charts+pull-request+integration-metricbeat.yml index fe53b9a70..34c6ac2fb 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+integration-metricbeat.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+integration-metricbeat.yml @@ -25,8 +25,11 @@ #!/usr/local/bin/runbld set -euo pipefail + source /usr/local/bin/bash_standard_lib.sh + set +x - export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x From 2eb8641c4b72ba41eba93c0efc4c455facb0d468 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Wed, 10 Jul 2019 16:04:11 +0200 Subject: [PATCH 092/115] Update .ci/jobs/elastic+helm-charts+master+integration-elasticsearch.yml Co-Authored-By: Tyler Langlois --- .../elastic+helm-charts+master+integration-elasticsearch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/jobs/elastic+helm-charts+master+integration-elasticsearch.yml b/.ci/jobs/elastic+helm-charts+master+integration-elasticsearch.yml index 5b5ccc95c..7bb9ea7f6 100644 --- a/.ci/jobs/elastic+helm-charts+master+integration-elasticsearch.yml +++ b/.ci/jobs/elastic+helm-charts+master+integration-elasticsearch.yml @@ -28,7 +28,7 @@ source /usr/local/bin/bash_standard_lib.sh set +x - VAULT_TOKEN=$(retry 5 retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") export VAULT_TOKEN unset VAULT_ROLE_ID VAULT_SECRET_ID set -x From 2c3a7ba0b2cbaa58453c7ebbf3d672937a391b2b Mon Sep 17 00:00:00 2001 From: Joris Andrade Date: Wed, 10 Jul 2019 16:58:47 +0200 Subject: [PATCH 093/115] Add option to disable sysctlInitContainer Signed-off-by: Joris Andrade --- elasticsearch/README.md | 3 ++- elasticsearch/templates/statefulset.yaml | 2 ++ elasticsearch/tests/elasticsearch_test.py | 4 ++++ elasticsearch/values.yaml | 3 +++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 22ff400e3..2096e3a68 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -108,7 +108,8 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 --set im | `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Elasticsearch service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | | `schedulerName` | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods) | `nil` | | `masterTerminationFix` | A workaround needed for Elasticsearch < 7.2.0 to prevent master status being lost during restarts [#63](https://github.com/elastic/helm-charts/issues/63) | `false` | -| `lifecycle` | Allows you to add lifecycle configuration. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | +| `lifecycle` | Allows you to add lifecycle configuration. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | +| `sysctlInitContainer.enabled` | Allows you to disable the sysctlInitContainer if you are not allowed to run privileged:true. | `false` | ## Try it out diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 46ec24dbc..b085dade4 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -116,6 +116,7 @@ spec: {{ toYaml .Values.imagePullSecrets | indent 8 }} {{- end }} initContainers: + {{- if .Values.sysctlInitContainer.enabled }} - name: configure-sysctl securityContext: runAsUser: 0 @@ -124,6 +125,7 @@ spec: command: ["sysctl", "-w", "vm.max_map_count={{ .Values.sysctlVmMaxMapCount}}"] resources: {{ toYaml .Values.initResources | indent 10 }} + {{- end }} {{- if .Values.extraInitContainers }} {{ tpl .Values.extraInitContainers . | indent 6 }} {{- end }} diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 1782d6802..6f4950979 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -82,6 +82,10 @@ def test_defaults(): 'name': 'node.ingest', 'value': 'true' }, + { + 'name': 'sysctlInitContainer.enabled', + 'value': 'true' + }, ] c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 4c41f87d6..6e53ee60d 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -204,3 +204,6 @@ lifecycle: {} # postStart: # exec: # command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"] + +sysctlInitContainer: + enabled: true From 1a10ea1680eb58b55c0b91bed394bb17919e6572 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Thu, 11 Jul 2019 11:51:51 +0200 Subject: [PATCH 094/115] Fixup the instructions for how to actual interact with the deployment --- elasticsearch/README.md | 3 +++ elasticsearch/examples/security/Makefile | 2 +- kibana/README.md | 12 ++++++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 6cbf43a2a..8ff9fa15b 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -140,6 +140,9 @@ A cluster with node to node security and https enabled. This example uses autoge ``` cd examples/security make + + # Run a curl command to interact with the cluster + kubectl exec -ti security-master-0 -- sh -c 'curl -u $ELASTIC_USERNAME:$ELASTIC_PASSWORD -k https://localhost:9200/_cluster/health?pretty' ``` ### FAQ diff --git a/elasticsearch/examples/security/Makefile b/elasticsearch/examples/security/Makefile index a2d447a22..827bdc56c 100644 --- a/elasticsearch/examples/security/Makefile +++ b/elasticsearch/examples/security/Makefile @@ -16,7 +16,7 @@ test: secrets install goss secrets: docker rm -f elastic-helm-charts-certs || true rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12 || true - password=$$([ ! -z "$$ELASTIC_PASSWORD" ] && echo $$ELASTIC_PASSWORD || echo $$(docker run --rm docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) /bin/sh -c "< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c20")) && \ + password=$$([ ! -z "$$ELASTIC_PASSWORD" ] && echo $$ELASTIC_PASSWORD || echo $$(docker run --rm docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) /bin/sh -c "< /dev/urandom tr -cd '[:alnum:]' | head -c20")) && \ docker run --name elastic-helm-charts-certs -i -w /app \ docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) \ /bin/sh -c " \ diff --git a/kibana/README.md b/kibana/README.md index 60427ed0c..1fc8845c4 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -96,9 +96,17 @@ In [examples/](./examples) you will find some example configurations. These exam ``` * Setup a port forward and access Kibana at https://localhost:5601 ``` + # Setup the port forward + kubectl port-forward deployment/helm-kibana-security-kibana 5601 + + # Run this in a seperate terminal # Get the auto generated password - kubectl get secret elastic-credentials -o jsonpath='{.data.password}' | base64 --decode - kubectl port-forward deployment/helm-kibana-default-kibana 5601 + password=$(kubectl get secret elastic-credentials -o jsonpath='{.data.password}' | base64 --decode) + echo $password + + # Test Kibana is working with curl or access it with your browser at https://localhost:5601 + # The example certificate is self signed so you may see a warning about the certificate + curl -I -k -u elastic:$password https://localhost:5601/app/kibana ``` ## Testing From 2188b4ad92d543a240cc516b2d3af459698b3240 Mon Sep 17 00:00:00 2001 From: Joris Andrade Date: Thu, 11 Jul 2019 15:01:43 +0200 Subject: [PATCH 095/115] Add tests and fix README.md Signed-off-by: Joris Andrade --- elasticsearch/README.md | 4 ++-- elasticsearch/tests/elasticsearch_test.py | 21 +++++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 2096e3a68..3209e3f67 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -99,6 +99,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 --set im | `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000` | | `securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) for the container | `capabilities.drop:[ALL]`
`runAsNonRoot: true`
`runAsUser: 1000` | | `terminationGracePeriod` | The [terminationGracePeriod](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods) in seconds used when trying to stop the pod | `120` | +| `sysctlInitContainer.enabled` | Allows you to disable the sysctlInitContainer if you are setting vm.max_map_count with another method | `true` | | `sysctlVmMaxMapCount` | Sets the [sysctl vm.max_map_count](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html#vm-max-map-count) needed for Elasticsearch | `262144` | | `readinessProbe` | Configuration fields for the [readinessProbe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | | `clusterHealthCheckParams` | The [Elasticsearch cluster health status params](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#request-params) that will be used by readinessProbe command | `wait_for_status=green&timeout=1s` | @@ -108,8 +109,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 --set im | `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Elasticsearch service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | | `schedulerName` | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods) | `nil` | | `masterTerminationFix` | A workaround needed for Elasticsearch < 7.2.0 to prevent master status being lost during restarts [#63](https://github.com/elastic/helm-charts/issues/63) | `false` | -| `lifecycle` | Allows you to add lifecycle configuration. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | -| `sysctlInitContainer.enabled` | Allows you to disable the sysctlInitContainer if you are not allowed to run privileged:true. | `false` | +| `lifecycle` | Allows you to add lifecycle configuration. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | ## Try it out diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 6f4950979..66b514104 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -82,10 +82,6 @@ def test_defaults(): 'name': 'node.ingest', 'value': 'true' }, - { - 'name': 'sysctlInitContainer.enabled', - 'value': 'true' - }, ] c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] @@ -338,6 +334,23 @@ def test_adding_a_extra_init_container(): extraInitContainer = r['statefulset'][uname]['spec']['template']['spec']['initContainers'] assert {'name': 'do-something', 'image': 'busybox', 'command': ['do', 'something'], } in extraInitContainer +def test_sysctl_init_container_disabled(): + config = ''' +sysctlInitContainer: + enabled: false +''' + r = helm_template(config) + initContainers = r['statefulset'][uname]['spec']['template']['spec']['initContainers'] + assert initContainers is None + +def test_sysctl_init_container_enabled(): + config = ''' +sysctlInitContainer: + enabled: true +''' + r = helm_template(config) + initContainers = r['statefulset'][uname]['spec']['template']['spec']['initContainers'] + assert initContainers[0]['name'] == 'configure-sysctl' def test_adding_storageclass_annotation_to_volumeclaimtemplate(): config = ''' From 1729759f22f1d6018335979b2881476a0b214251 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Wed, 17 Jul 2019 14:52:33 +0200 Subject: [PATCH 096/115] 7.2.1-0 Release Intermediate release to get the metricbeat chart live along wiht some other nice changes. * Dropped the `--version` specifier in the readmes. This was only needed back when we had the alpha flag because otherwise `helm install elastic/elasticsearch` failed because the helm client ignored these versions. --- CHANGELOG.md | 31 ++++++++++++++++++++++++++++--- README.md | 2 +- elasticsearch/Chart.yaml | 2 +- elasticsearch/README.md | 4 ++-- filebeat/Chart.yaml | 2 +- filebeat/README.md | 4 ++-- helpers/bumper.py | 9 +++++++-- helpers/release.md | 2 +- kibana/Chart.yaml | 2 +- kibana/README.md | 4 ++-- metricbeat/Chart.yaml | 2 +- metricbeat/README.md | 4 ++-- 12 files changed, 49 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a72412b6..63763897c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,34 @@ -## Unreleased - XXXX/XX/XX +# Changelog -### Metricbeat (WIP) +--- +## 7.2.1-0 - 2019/07/01 + +* [#195](https://github.com/elastic/helm-charts/pull/195) - @cclauss - Initial steps started to move all python2 code to python3 +* [#205](https://github.com/elastic/helm-charts/pull/205) - @Crazybus - Fixup and improve security example documentation + + +### Elasticsearch + +* [#171](https://github.com/elastic/helm-charts/pull/171) - @naseemkullah - Run Elasticsearch as a non-root user +* [#197](https://github.com/elastic/helm-charts/pull/197) - @tetianakravchenko - Add option to provide custom start/stop hooks +* [#206](https://github.com/elastic/helm-charts/pull/206) - @Crazybus - Automatically detect esMajorVersion for default images +* [#203](https://github.com/elastic/helm-charts/pull/203) - @Crazybus - Add testing for security context +* [#220](https://github.com/elastic/helm-charts/pull/220) - @JorisAndrade - Add option to disable sysctlInitContainer + +### Kibana + +* [#204](https://github.com/elastic/helm-charts/pull/204) - @Crazybus - Make imagePullPolicy actually do something +* [#210](https://github.com/elastic/helm-charts/pull/210) - @cliedeman - Add Kibana pod annotations +* [#217](https://github.com/elastic/helm-charts/pull/217) - @Crazybus - Update healthCheckPath to mention basePath usage + +### Filebeat + +* [#214](https://github.com/elastic/helm-charts/pull/214) - @dugouchet - Add additional labels + +### Metricbeat +* [#127](https://github.com/elastic/helm-charts/pull/127) - @Crazybus - Add metricbeat chart * [#128](https://github.com/elastic/helm-charts/pull/128) - @Crazybus - Add ci jobs for metricbeat -* [#127](https://github.com/elastic/helm-charts/pull/127) - @Crazybus - WIP add metricbeat chart --- ## 7.2.0 - 2019/07/01 diff --git a/README.md b/README.md index bceddafa8..f1ede78cf 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build Status](https://img.shields.io/jenkins/s/https/devops-ci.elastic.co/job/elastic+helm-charts+master.svg)](https://devops-ci.elastic.co/job/elastic+helm-charts+master/) -This functionality is in beta status and may be changed or removed completely in a future release. Elastic will take a best effort approach to fix any issues, but beta features are not subject to the support SLA of official GA features. +This functionality is in beta and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features. ## Charts diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml index 18077d545..930f30d81 100755 --- a/elasticsearch/Chart.yaml +++ b/elasticsearch/Chart.yaml @@ -4,7 +4,7 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: elasticsearch -version: 7.2.0 +version: 7.2.1-0 appVersion: 7.2.0 sources: - https://github.com/elastic/elasticsearch diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 9a4bdd70f..7edf2b181 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -34,7 +34,7 @@ If you currently have a cluster deployed with the [helm/charts stable](https://g ``` * Install it ``` - helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 + helm install --name elasticsearch elastic/elasticsearch ``` ## Compatibility @@ -50,7 +50,7 @@ Examples of installing older major versions can be found in the [examples](./exa While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.2.0` of Elasticsearch it would look like this: ``` -helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 --set imageTag=7.2.0 +helm install --name elasticsearch elastic/elasticsearch --set imageTag=7.2.0 ``` ## Configuration diff --git a/filebeat/Chart.yaml b/filebeat/Chart.yaml index 7b30bac13..d7dddb40d 100755 --- a/filebeat/Chart.yaml +++ b/filebeat/Chart.yaml @@ -4,7 +4,7 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: filebeat -version: 7.2.0 +version: 7.2.1-0 appVersion: 7.2.0 sources: - https://github.com/elastic/beats diff --git a/filebeat/README.md b/filebeat/README.md index 2870336ce..36eb229a5 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -21,7 +21,7 @@ This helm chart is a lightweight way to configure and run our official [Filebeat ``` * Install it ``` - helm install --name filebeat elastic/filebeat --version 7.2.0 + helm install --name filebeat elastic/filebeat ``` ## Compatibility @@ -37,7 +37,7 @@ Examples of installing older major versions can be found in the [examples](./exa While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.2.0` of Filebeat it would look like this: ``` -helm install --name filebeat elastic/filebeat --version 7.2.0 --set imageTag=7.2.0 +helm install --name filebeat elastic/filebeat --set imageTag=7.2.0 ``` diff --git a/helpers/bumper.py b/helpers/bumper.py index 976c88e1f..294e57f6f 100755 --- a/helpers/bumper.py +++ b/helpers/bumper.py @@ -7,6 +7,8 @@ os.chdir(os.path.join(os.path.dirname(__file__), '..')) +chart_version = '7.2.1-0' + versions = { 6: '6.8.1', 7: '7.2.0', @@ -26,7 +28,7 @@ blacklist = re.compile(r".*127.0.0.1.*") for major, version in versions.iteritems(): - r = re.compile(r"{0}\.[0-9]*\.[0-9]*".format(major)) + r = re.compile(r"^({0})\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$".format(major)) for pattern in file_patterns: for f in glob.glob(pattern): print(f) @@ -34,4 +36,7 @@ if re.match(blacklist, line): print(line.rstrip()) else: - print(r.sub(version, line.rstrip())) + if f.endswith('Chart.yaml') and line.startswith('version:'): + print(r.sub(chart_version, line.rstrip())) + else: + print(r.sub(version, line.rstrip())) diff --git a/helpers/release.md b/helpers/release.md index 2d99ffc2e..590323a82 100644 --- a/helpers/release.md +++ b/helpers/release.md @@ -1,7 +1,7 @@ # Release process * Update the [changelog](/CHANGELOG.md) -* Update the stack versions in [bumper.py](/helpers/bumper.py) and run the script. This will update the versions in all the right places +* Update the stack and chart versions in [bumper.py](/helpers/bumper.py) and run the script. This will update the versions in all the right places * Open a pull request and wait for a green build before merging * Create a [new release](https://github.com/elastic/helm-charts/releases/new) and include the latest changelog entry * Run the [release script](/helpers/release.py) to build and upload the artifact diff --git a/kibana/Chart.yaml b/kibana/Chart.yaml index 2095e37f6..09a985f33 100755 --- a/kibana/Chart.yaml +++ b/kibana/Chart.yaml @@ -4,7 +4,7 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: kibana -version: 7.2.0 +version: 7.2.1-0 appVersion: 7.2.0 sources: - https://github.com/elastic/kibana diff --git a/kibana/README.md b/kibana/README.md index 873907e6a..17aa2b3f0 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -17,7 +17,7 @@ This helm chart is a lightweight way to configure and run our official [Kibana d ``` * Install it ``` - helm install --name kibana elastic/kibana --version 7.2.0 + helm install --name kibana elastic/kibana ``` ## Compatibility @@ -33,7 +33,7 @@ Examples of installing older major versions can be found in the [examples](./exa While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.2.0` of Kibana it would look like this: ``` -helm install --name kibana elastic/kibana --version 7.2.0 --set imageTag=7.2.0 +helm install --name kibana elastic/kibana --set imageTag=7.2.0 ``` ## Configuration diff --git a/metricbeat/Chart.yaml b/metricbeat/Chart.yaml index debbbe01c..ba5db384f 100755 --- a/metricbeat/Chart.yaml +++ b/metricbeat/Chart.yaml @@ -4,7 +4,7 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: metricbeat -version: 7.2.0 +version: 7.2.1-0 appVersion: 7.2.0 sources: - https://github.com/elastic/beats diff --git a/metricbeat/README.md b/metricbeat/README.md index 9c728ad8b..5f2dd02bd 100644 --- a/metricbeat/README.md +++ b/metricbeat/README.md @@ -17,7 +17,7 @@ This helm chart is a lightweight way to configure and run our official [Metricbe ``` * Install it ``` - helm install --name metricbeat elastic/metricbeat --version 7.2.0 + helm install --name metricbeat elastic/metricbeat ``` ## Compatibility @@ -33,7 +33,7 @@ Examples of installing older major versions can be found in the [examples](./exa While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.2.0` of metricbeat it would look like this: ``` -helm install --name metricbeat elastic/metricbeat --version 7.2.0 --set imageTag=7.2.0 +helm install --name metricbeat elastic/metricbeat --set imageTag=7.2.0 ``` From 58f8f501c10bc3095dc10e4bdce81126cafc2938 Mon Sep 17 00:00:00 2001 From: Michael Marie-Julie Date: Wed, 17 Jul 2019 18:12:07 +0200 Subject: [PATCH 097/115] [elasticsearch] additionals labels --- elasticsearch/README.md | 1 + elasticsearch/templates/statefulset.yaml | 3 +++ elasticsearch/tests/elasticsearch_test.py | 8 ++++++++ elasticsearch/values.yaml | 3 +++ 4 files changed, 15 insertions(+) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 9a4bdd70f..818d2e4c9 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -74,6 +74,7 @@ helm install --name elasticsearch elastic/elasticsearch --version 7.2.0 --set im | `imageTag` | The Elasticsearch docker image tag | `7.2.0` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Elasticsearch pods | `{}` | +| `labels` | Configurable [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) applied to all Elasticsearch pods | `{}` | | `esJavaOpts` | [Java options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for Elasticsearch. This is where you should configure the [jvm heap size](https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html) | `-Xmx1g -Xms1g` | | `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 2Gi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | | `initResources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the initContainer in the statefulset | {} | diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index b085dade4..30dcec51e 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -8,6 +8,9 @@ metadata: release: {{ .Release.Name | quote }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" app: "{{ template "uname" . }}" + {{- range $key, $value := .Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} annotations: esMajorVersion: "{{ include "esMajorVersion" . }}" spec: diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 66b514104..cdc08a79a 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -785,3 +785,11 @@ def test_set_container_security_context(): assert c['securityContext']['runAsNonRoot'] == True assert c['securityContext']['runAsUser'] == 1001 assert c['securityContext']['other'] == 'test' + +def test_adding_pod_labels(): + config = ''' +labels: + app.kubernetes.io/name: elasticsearch +''' + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['labels']['app.kubernetes.io/name'] == 'elasticsearch' diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 6e53ee60d..9caf0bcce 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -49,6 +49,9 @@ imagePullPolicy: "IfNotPresent" podAnnotations: {} # iam.amazonaws.com/role: es-cluster +# additionals labels +labels: {} + esJavaOpts: "-Xmx1g -Xms1g" resources: From e92e910831b57c3615942ead259f411e230ef98d Mon Sep 17 00:00:00 2001 From: Olivier Lebhard Date: Wed, 17 Jul 2019 17:26:18 +0200 Subject: [PATCH 098/115] [kibana] - Add additionals labels --- kibana/README.md | 1 + kibana/templates/deployment.yaml | 3 +++ kibana/tests/kibana_test.py | 10 +++++++++- kibana/values.yaml | 3 +++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/kibana/README.md b/kibana/README.md index 873907e6a..19bdc017c 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -69,6 +69,7 @@ helm install --name kibana elastic/kibana --version 7.2.0 --set imageTag=7.2.0 | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | | `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Kibana service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | | `service` | Configurable [service](https://kubernetes.io/docs/concepts/services-networking/service/) to expose the Kibana service. See [`values.yaml`](./values.yaml) for an example | `type: ClusterIP`
`port: 5601`
`nodePort:`
`annotations: {}` | +| `labels` | Configurable [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) applied to all Kibana pods | `{}` | ## Examples diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml index e31f5c6e5..95616ec0a 100644 --- a/kibana/templates/deployment.yaml +++ b/kibana/templates/deployment.yaml @@ -5,6 +5,9 @@ metadata: labels: app: {{ .Chart.Name }} release: {{ .Release.Name | quote }} + {{- range $key, $value := .Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} spec: replicas: {{ .Values.replicas }} strategy: diff --git a/kibana/tests/kibana_test.py b/kibana/tests/kibana_test.py index b85315f4a..58b92f440 100644 --- a/kibana/tests/kibana_test.py +++ b/kibana/tests/kibana_test.py @@ -355,4 +355,12 @@ def test_override_imagePullPolicy(): r = helm_template(config) c = r['deployment'][name]['spec']['template']['spec']['containers'][0] - assert c['imagePullPolicy'] == 'Always' \ No newline at end of file + assert c['imagePullPolicy'] == 'Always' + + def test_adding_pod_labels(): + config = ''' +labels: + app.kubernetes.io/name: kibana +''' + r = helm_template(config) + assert r['deployment'][name]['metadata']['labels']['app.kubernetes.io/name'] == 'kibana' \ No newline at end of file diff --git a/kibana/values.yaml b/kibana/values.yaml index e34407e23..cf08ef6f8 100755 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -24,6 +24,9 @@ image: "docker.elastic.co/kibana/kibana" imageTag: "7.2.0" imagePullPolicy: "IfNotPresent" +# additionals labels +labels: {} + podAnnotations: {} # iam.amazonaws.com/role: es-cluster From 3982fdb35955426909d096bc6fc5f9be9a91c72f Mon Sep 17 00:00:00 2001 From: Olivier Lebhard Date: Fri, 19 Jul 2019 00:30:27 +0200 Subject: [PATCH 099/115] Update kibana/tests/kibana_test.py Co-Authored-By: Michael Russell --- kibana/tests/kibana_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kibana/tests/kibana_test.py b/kibana/tests/kibana_test.py index 58b92f440..e3453b615 100644 --- a/kibana/tests/kibana_test.py +++ b/kibana/tests/kibana_test.py @@ -357,10 +357,10 @@ def test_override_imagePullPolicy(): c = r['deployment'][name]['spec']['template']['spec']['containers'][0] assert c['imagePullPolicy'] == 'Always' - def test_adding_pod_labels(): +def test_adding_pod_labels(): config = ''' labels: app.kubernetes.io/name: kibana ''' r = helm_template(config) - assert r['deployment'][name]['metadata']['labels']['app.kubernetes.io/name'] == 'kibana' \ No newline at end of file + assert r['deployment'][name]['metadata']['labels']['app.kubernetes.io/name'] == 'kibana' From df7118b48d9a410f38a6b66328a2fdf4ce1e5d04 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Mon, 22 Jul 2019 17:02:34 +0200 Subject: [PATCH 100/115] [kibana] Add subPath support to secretMounts Fixes: #229 This is needed in situation where you need to mount a specific secret as a file in a directory that contains other files. This was added for the Elasticsearch chart to make it possible to mount the keystore, the same option is needed for Kibana too as seen in in #229. --- kibana/templates/deployment.yaml | 3 +++ kibana/tests/kibana_test.py | 30 ++++++++++++++++++++++++++++++ kibana/values.yaml | 7 ++++--- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml index 95616ec0a..a0310b959 100644 --- a/kibana/templates/deployment.yaml +++ b/kibana/templates/deployment.yaml @@ -112,6 +112,9 @@ spec: {{- range .Values.secretMounts }} - name: {{ .name }} mountPath: {{ .path }} + {{- if .subPath }} + subPath: {{ .subPath }} + {{- end }} {{- end }} {{- range $path, $config := .Values.kibanaConfig }} - name: kibanaconfig diff --git a/kibana/tests/kibana_test.py b/kibana/tests/kibana_test.py index e3453b615..9b57f4c53 100644 --- a/kibana/tests/kibana_test.py +++ b/kibana/tests/kibana_test.py @@ -364,3 +364,33 @@ def test_adding_pod_labels(): ''' r = helm_template(config) assert r['deployment'][name]['metadata']['labels']['app.kubernetes.io/name'] == 'kibana' + +def test_adding_a_secret_mount_with_subpath(): + config = ''' +secretMounts: + - name: elastic-certificates + secretName: elastic-certs + path: /usr/share/elasticsearch/config/certs + subPath: cert.crt +''' + r = helm_template(config) + d = r['deployment'][name]['spec']['template']['spec'] + assert d['containers'][0]['volumeMounts'][-1] == { + 'mountPath': '/usr/share/elasticsearch/config/certs', + 'subPath': 'cert.crt', + 'name': 'elastic-certificates' + } + +def test_adding_a_secret_mount_without_subpath(): + config = ''' +secretMounts: + - name: elastic-certificates + secretName: elastic-certs + path: /usr/share/elasticsearch/config/certs +''' + r = helm_template(config) + d = r['deployment'][name]['spec']['template']['spec'] + assert d['containers'][0]['volumeMounts'][-1] == { + 'mountPath': '/usr/share/elasticsearch/config/certs', + 'name': 'elastic-certificates' + } diff --git a/kibana/values.yaml b/kibana/values.yaml index cf08ef6f8..e253b3e66 100755 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -16,9 +16,10 @@ extraEnvs: [] # This is useful for mounting certificates for security and for mounting # the X-Pack license secretMounts: [] -# - name: elastic-certificates -# secretName: elastic-certificates -# path: /usr/share/elasticsearch/config/certs +# - name: kibana-keystore +# secretName: kibana-keystore +# path: /usr/share/kibana/data/kibana.keystore +# subPath: kibana.keystore # optional image: "docker.elastic.co/kibana/kibana" imageTag: "7.2.0" From d419529f43c24277403e68fa73dd5e3f5a330bd0 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 29 Jul 2019 13:36:22 +0200 Subject: [PATCH 101/115] publishNotReadyAddresses not working according to https://github.com/kubernetes/kubernetes/issues/58662 the `publishNotReadyAddresses: true` is not working. Instead they suggest this annotation as workaround: `service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"` --- elasticsearch/templates/service.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elasticsearch/templates/service.yaml b/elasticsearch/templates/service.yaml index 474e8f919..8342bb65e 100644 --- a/elasticsearch/templates/service.yaml +++ b/elasticsearch/templates/service.yaml @@ -37,6 +37,8 @@ metadata: release: {{ .Release.Name | quote }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" app: "{{ template "uname" . }}" + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" spec: clusterIP: None # This is needed for statefulset hostnames like elasticsearch-0 to resolve # Create endpoints also if the related pod isn't ready From 549497b1120a02fc35e6d7e167784b7e6589d2c5 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Wed, 31 Jul 2019 15:57:57 +0200 Subject: [PATCH 102/115] 7.3.0 Release Also fixed up the markdown formatting for tables --- CHANGELOG.md | 18 ++- README.md | 11 +- elasticsearch/Chart.yaml | 4 +- elasticsearch/README.md | 118 +++++++++--------- elasticsearch/examples/default/test/goss.yaml | 2 +- elasticsearch/examples/oss/test/goss.yaml | 2 +- elasticsearch/examples/upgrade/test/goss.yaml | 2 +- elasticsearch/values.yaml | 2 +- filebeat/Chart.yaml | 4 +- filebeat/README.md | 12 +- filebeat/examples/default/test/goss.yaml | 4 +- filebeat/examples/oss/test/goss.yaml | 2 +- filebeat/examples/security/test/goss.yaml | 2 +- filebeat/values.yaml | 2 +- helpers/bumper.py | 6 +- helpers/examples.mk | 2 +- kibana/Chart.yaml | 4 +- kibana/README.md | 10 +- kibana/examples/default/test/goss.yaml | 2 +- kibana/values.yaml | 2 +- metricbeat/Chart.yaml | 4 +- metricbeat/README.md | 8 +- .../examples/default/test/goss-metrics.yaml | 6 +- metricbeat/examples/default/test/goss.yaml | 6 +- .../examples/oss/test/goss-metrics.yaml | 6 +- metricbeat/examples/oss/test/goss.yaml | 6 +- .../examples/security/test/goss-metrics.yaml | 6 +- metricbeat/examples/security/test/goss.yaml | 6 +- metricbeat/values.yaml | 2 +- 29 files changed, 140 insertions(+), 121 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63763897c..a5abeea73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,23 @@ # Changelog +## 7.3.0 - 2019/07/31 + +* 7.3.0 as the default stack version + +### Elasticsearch +| PR | Author | Title | +| ------------------------------------------------------- | ---------------------------------------------------------- | ------------------------------------------------------------------------- | +| [#226](https://github.com/elastic/helm-charts/pull/226) | [@MichaelMarieJulie](https://github.com/MichaelMarieJulie) | Add configurable pods labels | +| [#237](https://github.com/elastic/helm-charts/pull/237) | [@MichaelSp](https://github.com/MichaelSp) | Add back `service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"` | + +### Kibana +| PR | Author | Title | +| ------------------------------------------------------- | ------------------------------------------ | ----------------------------------- | +| [#225](https://github.com/elastic/helm-charts/pull/225) | [@plumcraft](https://github.com/plumcraft) | Add configurable pod labels | +| [#230](https://github.com/elastic/helm-charts/pull/230) | [@Crazybus](https://github.com/Crazybus) | Add subPath support to secretMounts | + --- -## 7.2.1-0 - 2019/07/01 +## 7.2.1-0 - 2019/07/18 * [#195](https://github.com/elastic/helm-charts/pull/195) - @cclauss - Initial steps started to move all python2 code to python3 * [#205](https://github.com/elastic/helm-charts/pull/205) - @Crazybus - Fixup and improve security example documentation diff --git a/README.md b/README.md index f1ede78cf..1c2732d68 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,16 @@ This functionality is in beta and is subject to change. The design and code is l Please look in the chart directories for the documentation for each chart. These helm charts are designed to be a lightweight way to configure our official docker images. Links to the relevant docker image documentation has also been added below. -* [Elasticsearch](./elasticsearch/README.md) - [docker image docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html) -* [Kibana](./kibana/README.md) - [docker image docs](https://www.elastic.co/guide/en/kibana/current/docker.html) -* [Filebeat](./filebeat/README.md) - [docker image docs](https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html) +| Chart | Docker documentation | +| ------------------------------------------ | ------------------------------------------------------------------------------- | +| [Elasticsearch](./elasticsearch/README.md) | https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html | +| [Kibana](./kibana/README.md) | https://www.elastic.co/guide/en/kibana/current/docker.html | +| [Filebeat](./filebeat/README.md) | https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html | +| [Metricbeat](./metricbeat/README.md) | https://www.elastic.co/guide/en/beats/metricbeat/current/running-on-docker.html | ## Kubernetes versions -The charts are [currently tested](https://devops-ci.elastic.co/job/elastic+helm-charts+master/) against all GKE versions available. +The charts are [currently tested](https://devops-ci.elastic.co/job/elastic+helm-charts+master/) against all GKE versions available. The exact versions are defined under `KUBERNETES_VERSIONS` in [helpers/matrix.yml](/helpers/matrix.yml) diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml index 930f30d81..7a4f95cd1 100755 --- a/elasticsearch/Chart.yaml +++ b/elasticsearch/Chart.yaml @@ -4,8 +4,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: elasticsearch -version: 7.2.1-0 -appVersion: 7.2.0 +version: 7.3.0 +appVersion: 7.3.0 sources: - https://github.com/elastic/elasticsearch icon: https://helm.elastic.co/icons/elasticsearch.png diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 860bf3d6d..924800a48 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -43,74 +43,74 @@ This chart is tested with the latest supported versions. The currently tested ve | 6.x | 7.x | | ----- | ----- | -| 6.8.1 | 7.2.0 | +| 6.8.1 | 7.3.0 | Examples of installing older major versions can be found in the [examples](./examples) directory. -While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.2.0` of Elasticsearch it would look like this: +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.3.0` of Elasticsearch it would look like this: ``` -helm install --name elasticsearch elastic/elasticsearch --set imageTag=7.2.0 +helm install --name elasticsearch elastic/elasticsearch --set imageTag=7.3.0 ``` ## Configuration -| Parameter | Description | Default | -| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | -| `clusterName` | This will be used as the Elasticsearch [cluster.name](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.name.html) and should be unique per cluster in the namespace | `elasticsearch` | -| `nodeGroup` | This is the name that will be used for each group of nodes in the cluster. The name will be `clusterName-nodeGroup-X` | `master` | -| `masterService` | Optional. The service name used to connect to the masters. You only need to set this if your master `nodeGroup` is set to something other than `master`. See [Clustering and Node Discovery](#clustering-and-node-discovery) for more information. | `` | -| `roles` | A hash map with the [specific roles](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) for the node group | `master: true`
`data: true`
`ingest: true` | -| `replicas` | Kubernetes replica count for the statefulset (i.e. how many pods) | `3` | -| `minimumMasterNodes` | The value for [discovery.zen.minimum_master_nodes](https://www.elastic.co/guide/en/elasticsearch/reference/6.7/discovery-settings.html#minimum_master_nodes). Should be set to `(master_eligible_nodes / 2) + 1`. Ignored in Elasticsearch versions >= 7. | `2` | -| `esMajorVersion` | Used to set major version specific configuration. If you are using a custom image and not running the default Elasticsearch version you will need to set this to the version you are running (e.g. `esMajorVersion: 6`) | `""` | -| `esConfig` | Allows you to add any config files in `/usr/share/elasticsearch/config/` such as `elasticsearch.yml` and `log4j2.properties`. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | -| `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | -| `extraVolumes` | Additional volumes to be passed to the `tpl` function | | -| `extraVolumeMounts` | Additional volumeMounts to be passed to the `tpl` function | | -| `extraInitContainers` | Additional init containers to be passed to the `tpl` function | | -| `secretMounts` | Allows you easily mount a secret as a file inside the statefulset. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | -| `image` | The Elasticsearch docker image | `docker.elastic.co/elasticsearch/elasticsearch` | -| `imageTag` | The Elasticsearch docker image tag | `7.2.0` | -| `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | -| `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Elasticsearch pods | `{}` | -| `labels` | Configurable [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) applied to all Elasticsearch pods | `{}` | -| `esJavaOpts` | [Java options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for Elasticsearch. This is where you should configure the [jvm heap size](https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html) | `-Xmx1g -Xms1g` | -| `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 2Gi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | -| `initResources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the initContainer in the statefulset | {} | -| `sidecarResources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the sidecar containers in the statefulset | {} | -| `networkHost` | Value for the [network.host Elasticsearch setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/network.host.html) | `0.0.0.0` | -| `volumeClaimTemplate` | Configuration for the [volumeClaimTemplate for statefulsets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-storage). You will want to adjust the storage (default `30Gi`) and the `storageClassName` if you are using a different storage class | `accessModes: [ "ReadWriteOnce" ]`
`resources.requests.storage: 30Gi` | -| `persistence.annotations` | Additional persistence annotations for the `volumeClaimTemplate` | `{}` | -| `persistence.enabled` | Enables a persistent volume for Elasticsearch data. Can be disabled for nodes that only have [roles](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) which don't require persistent data. | `true` | -| `priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `` | -| `antiAffinityTopologyKey` | The [anti-affinity topology key](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). By default this will prevent multiple Elasticsearch nodes from running on the same Kubernetes node | `kubernetes.io/hostname` | -| `antiAffinity` | Setting this to hard enforces the [anti-affinity rules](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). If it is set to soft it will be done "best effort". Other values will be ignored. | `hard` | -| `nodeAffinity` | Value for the [node affinity settings](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature) | `{}` | -| `podManagementPolicy` | By default Kubernetes [deploys statefulsets serially](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies). This deploys them in parallel so that they can discover eachother | `Parallel` | -| `protocol` | The protocol that will be used for the readinessProbe. Change this to `https` if you have `xpack.security.http.ssl.enabled` set | `http` | -| `httpPort` | The http port that Kubernetes will use for the healthchecks and the service. If you change this you will also need to set [http.port](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#_settings) in `extraEnvs` | `9200` | -| `transportPort` | The transport port that Kubernetes will use for the service. If you change this you will also need to set [transport port configuration](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#_transport_settings) in `extraEnvs` | `9300` | -| `service.type` | Type of elasticsearch service. [Service Types](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) | `ClusterIP` | -| `service.nodePort` | Custom [nodePort](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport) port that can be set if you are using `service.type: nodePort`. | `` | -| `service.annotations` | Annotations that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` [Annotations](https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws) | `{}` | -| `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) for the statefulset. By default Kubernetes will wait for the cluster to be green after upgrading each pod. Setting this to `OnDelete` will allow you to manually delete each pod during upgrades | `RollingUpdate` | -| `maxUnavailable` | The [maxUnavailable](https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget) value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group | `1` | -| `fsGroup (DEPRECATED)` | The Group ID (GID) for [securityContext.fsGroup](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) so that the Elasticsearch user can read from the persistent volume | `` | -| `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000` | -| `securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) for the container | `capabilities.drop:[ALL]`
`runAsNonRoot: true`
`runAsUser: 1000` | -| `terminationGracePeriod` | The [terminationGracePeriod](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods) in seconds used when trying to stop the pod | `120` | -| `sysctlInitContainer.enabled` | Allows you to disable the sysctlInitContainer if you are setting vm.max_map_count with another method | `true` | -| `sysctlVmMaxMapCount` | Sets the [sysctl vm.max_map_count](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html#vm-max-map-count) needed for Elasticsearch | `262144` | -| `readinessProbe` | Configuration fields for the [readinessProbe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | -| `clusterHealthCheckParams` | The [Elasticsearch cluster health status params](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#request-params) that will be used by readinessProbe command | `wait_for_status=green&timeout=1s` | -| `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | -| `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) so that you can target specific nodes for your Elasticsearch cluster | `{}` | -| `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | -| `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Elasticsearch service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | -| `schedulerName` | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods) | `nil` | -| `masterTerminationFix` | A workaround needed for Elasticsearch < 7.2.0 to prevent master status being lost during restarts [#63](https://github.com/elastic/helm-charts/issues/63) | `false` | -| `lifecycle` | Allows you to add lifecycle configuration. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | +| Parameter | Description | Default | +| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `clusterName` | This will be used as the Elasticsearch [cluster.name](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.name.html) and should be unique per cluster in the namespace | `elasticsearch` | +| `nodeGroup` | This is the name that will be used for each group of nodes in the cluster. The name will be `clusterName-nodeGroup-X` | `master` | +| `masterService` | Optional. The service name used to connect to the masters. You only need to set this if your master `nodeGroup` is set to something other than `master`. See [Clustering and Node Discovery](#clustering-and-node-discovery) for more information. | `` | +| `roles` | A hash map with the [specific roles](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) for the node group | `master: true`
`data: true`
`ingest: true` | +| `replicas` | Kubernetes replica count for the statefulset (i.e. how many pods) | `3` | +| `minimumMasterNodes` | The value for [discovery.zen.minimum_master_nodes](https://www.elastic.co/guide/en/elasticsearch/reference/6.7/discovery-settings.html#minimum_master_nodes). Should be set to `(master_eligible_nodes / 2) + 1`. Ignored in Elasticsearch versions >= 7. | `2` | +| `esMajorVersion` | Used to set major version specific configuration. If you are using a custom image and not running the default Elasticsearch version you will need to set this to the version you are running (e.g. `esMajorVersion: 6`) | `""` | +| `esConfig` | Allows you to add any config files in `/usr/share/elasticsearch/config/` such as `elasticsearch.yml` and `log4j2.properties`. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | +| `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | +| `extraVolumes` | Additional volumes to be passed to the `tpl` function | | +| `extraVolumeMounts` | Additional volumeMounts to be passed to the `tpl` function | | +| `extraInitContainers` | Additional init containers to be passed to the `tpl` function | | +| `secretMounts` | Allows you easily mount a secret as a file inside the statefulset. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | +| `image` | The Elasticsearch docker image | `docker.elastic.co/elasticsearch/elasticsearch` | +| `imageTag` | The Elasticsearch docker image tag | `7.3.0` | +| `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | +| `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Elasticsearch pods | `{}` | +| `labels` | Configurable [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) applied to all Elasticsearch pods | `{}` | +| `esJavaOpts` | [Java options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for Elasticsearch. This is where you should configure the [jvm heap size](https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html) | `-Xmx1g -Xms1g` | +| `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 2Gi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | +| `initResources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the initContainer in the statefulset | {} | +| `sidecarResources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the sidecar containers in the statefulset | {} | +| `networkHost` | Value for the [network.host Elasticsearch setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/network.host.html) | `0.0.0.0` | +| `volumeClaimTemplate` | Configuration for the [volumeClaimTemplate for statefulsets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-storage). You will want to adjust the storage (default `30Gi`) and the `storageClassName` if you are using a different storage class | `accessModes: [ "ReadWriteOnce" ]`
`resources.requests.storage: 30Gi` | +| `persistence.annotations` | Additional persistence annotations for the `volumeClaimTemplate` | `{}` | +| `persistence.enabled` | Enables a persistent volume for Elasticsearch data. Can be disabled for nodes that only have [roles](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) which don't require persistent data. | `true` | +| `priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `` | +| `antiAffinityTopologyKey` | The [anti-affinity topology key](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). By default this will prevent multiple Elasticsearch nodes from running on the same Kubernetes node | `kubernetes.io/hostname` | +| `antiAffinity` | Setting this to hard enforces the [anti-affinity rules](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). If it is set to soft it will be done "best effort". Other values will be ignored. | `hard` | +| `nodeAffinity` | Value for the [node affinity settings](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature) | `{}` | +| `podManagementPolicy` | By default Kubernetes [deploys statefulsets serially](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies). This deploys them in parallel so that they can discover eachother | `Parallel` | +| `protocol` | The protocol that will be used for the readinessProbe. Change this to `https` if you have `xpack.security.http.ssl.enabled` set | `http` | +| `httpPort` | The http port that Kubernetes will use for the healthchecks and the service. If you change this you will also need to set [http.port](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#_settings) in `extraEnvs` | `9200` | +| `transportPort` | The transport port that Kubernetes will use for the service. If you change this you will also need to set [transport port configuration](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#_transport_settings) in `extraEnvs` | `9300` | +| `service.type` | Type of elasticsearch service. [Service Types](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) | `ClusterIP` | +| `service.nodePort` | Custom [nodePort](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport) port that can be set if you are using `service.type: nodePort`. | `` | +| `service.annotations` | Annotations that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` [Annotations](https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws) | `{}` | +| `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) for the statefulset. By default Kubernetes will wait for the cluster to be green after upgrading each pod. Setting this to `OnDelete` will allow you to manually delete each pod during upgrades | `RollingUpdate` | +| `maxUnavailable` | The [maxUnavailable](https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget) value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group | `1` | +| `fsGroup (DEPRECATED)` | The Group ID (GID) for [securityContext.fsGroup](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) so that the Elasticsearch user can read from the persistent volume | `` | +| `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000` | +| `securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) for the container | `capabilities.drop:[ALL]`
`runAsNonRoot: true`
`runAsUser: 1000` | +| `terminationGracePeriod` | The [terminationGracePeriod](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods) in seconds used when trying to stop the pod | `120` | +| `sysctlInitContainer.enabled` | Allows you to disable the sysctlInitContainer if you are setting vm.max_map_count with another method | `true` | +| `sysctlVmMaxMapCount` | Sets the [sysctl vm.max_map_count](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html#vm-max-map-count) needed for Elasticsearch | `262144` | +| `readinessProbe` | Configuration fields for the [readinessProbe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | +| `clusterHealthCheckParams` | The [Elasticsearch cluster health status params](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#request-params) that will be used by readinessProbe command | `wait_for_status=green&timeout=1s` | +| `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | +| `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) so that you can target specific nodes for your Elasticsearch cluster | `{}` | +| `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | +| `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Elasticsearch service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | +| `schedulerName` | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods) | `nil` | +| `masterTerminationFix` | A workaround needed for Elasticsearch < 7.2 to prevent master status being lost during restarts [#63](https://github.com/elastic/helm-charts/issues/63) | `false` | +| `lifecycle` | Allows you to add lifecycle configuration. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | ## Try it out diff --git a/elasticsearch/examples/default/test/goss.yaml b/elasticsearch/examples/default/test/goss.yaml index 01ae5af40..d2c59dd7e 100644 --- a/elasticsearch/examples/default/test/goss.yaml +++ b/elasticsearch/examples/default/test/goss.yaml @@ -15,7 +15,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "7.2.0"' + - '"number" : "7.3.0"' - '"cluster_name" : "elasticsearch"' - '"name" : "elasticsearch-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/examples/oss/test/goss.yaml b/elasticsearch/examples/oss/test/goss.yaml index 63937ec96..769a6687f 100644 --- a/elasticsearch/examples/oss/test/goss.yaml +++ b/elasticsearch/examples/oss/test/goss.yaml @@ -11,7 +11,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "7.2.0"' + - '"number" : "7.3.0"' - '"cluster_name" : "oss"' - '"name" : "oss-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/examples/upgrade/test/goss.yaml b/elasticsearch/examples/upgrade/test/goss.yaml index b48364c03..ac71ba348 100644 --- a/elasticsearch/examples/upgrade/test/goss.yaml +++ b/elasticsearch/examples/upgrade/test/goss.yaml @@ -11,7 +11,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "7.2.0"' + - '"number" : "7.3.0"' - '"cluster_name" : "upgrade"' - '"name" : "upgrade-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 9caf0bcce..13ca0626d 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -43,7 +43,7 @@ secretMounts: [] # path: /usr/share/elasticsearch/config/certs image: "docker.elastic.co/elasticsearch/elasticsearch" -imageTag: "7.2.0" +imageTag: "7.3.0" imagePullPolicy: "IfNotPresent" podAnnotations: {} diff --git a/filebeat/Chart.yaml b/filebeat/Chart.yaml index d7dddb40d..8a653fac8 100755 --- a/filebeat/Chart.yaml +++ b/filebeat/Chart.yaml @@ -4,8 +4,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: filebeat -version: 7.2.1-0 -appVersion: 7.2.0 +version: 7.3.0 +appVersion: 7.3.0 sources: - https://github.com/elastic/beats icon: https://helm.elastic.co/icons/filebeat.png diff --git a/filebeat/README.md b/filebeat/README.md index 36eb229a5..7a6b3ec32 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -30,14 +30,14 @@ This chart is tested with the latest supported versions. The currently tested ve | 6.x | 7.x | | ----- | ----- | -| 6.8.1 | 7.2.0 | +| 6.8.1 | 7.3.0 | Examples of installing older major versions can be found in the [examples](./examples) directory. -While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.2.0` of Filebeat it would look like this: +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.3.0` of Filebeat it would look like this: ``` -helm install --name filebeat elastic/filebeat --set imageTag=7.2.0 +helm install --name filebeat elastic/filebeat --set imageTag=7.3.0 ``` @@ -50,13 +50,13 @@ helm install --name filebeat elastic/filebeat --set imageTag=7.2.0 | `extraVolumes` | Any extra volumes to define for the pod | `[]` | | `hostPathRoot` | Fully-qualified [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) that will be used to persist Filebeat registry data | `/var/lib` | | `image` | The Filebeat docker image | `docker.elastic.co/beats/filebeat` | -| `imageTag` | The Filebeat docker image tag | `7.2.0` | +| `imageTag` | The Filebeat docker image tag | `7.3.0` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | | `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles. | `true` | | `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Filebeat pods | `{}` | -| `labels` | Configurable [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) applied to all Filebeat pods | `{}` | -| `podSecurityContext` | Configurable [podSecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for Filebeat pod execution environment | `runAsUser: 0`
`privileged: false` | +| `labels` | Configurable [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) applied to all Filebeat pods | `{}` | +| `podSecurityContext` | Configurable [podSecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for Filebeat pod execution environment | `runAsUser: 0`
`privileged: false` | | `livenessProbe` | Parameters to pass to [liveness probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) checks for values such as timeouts and thresholds. | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | | `readinessProbe` | Parameters to pass to [readiness probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) checks for values such as timeouts and thresholds. | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | | `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the `DaemonSet` | `requests.cpu: 100m`
`requests.memory: 100Mi`
`limits.cpu: 1000m`
`limits.memory: 200Mi` | diff --git a/filebeat/examples/default/test/goss.yaml b/filebeat/examples/default/test/goss.yaml index 29d4cd6df..2de3a9fb0 100644 --- a/filebeat/examples/default/test/goss.yaml +++ b/filebeat/examples/default/test/goss.yaml @@ -29,7 +29,7 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-7.2.0' + - 'filebeat-7.3.0' file: /usr/share/filebeat/filebeat.yml: @@ -44,4 +44,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://elasticsearch-master:9200' - - 'version: 7.2.0' + - 'version: 7.3.0' diff --git a/filebeat/examples/oss/test/goss.yaml b/filebeat/examples/oss/test/goss.yaml index a40f18579..03b8907a9 100644 --- a/filebeat/examples/oss/test/goss.yaml +++ b/filebeat/examples/oss/test/goss.yaml @@ -19,4 +19,4 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-7.2.0' + - 'filebeat-7.3.0' diff --git a/filebeat/examples/security/test/goss.yaml b/filebeat/examples/security/test/goss.yaml index ce3f0f708..2df9e37ee 100644 --- a/filebeat/examples/security/test/goss.yaml +++ b/filebeat/examples/security/test/goss.yaml @@ -3,7 +3,7 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-7.2.0' + - 'filebeat-7.3.0' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 9cf900692..3da889cdd 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -34,7 +34,7 @@ extraVolumes: [] hostPathRoot: /var/lib image: "docker.elastic.co/beats/filebeat" -imageTag: "7.2.0" +imageTag: "7.3.0" imagePullPolicy: "IfNotPresent" imagePullSecrets: [] diff --git a/helpers/bumper.py b/helpers/bumper.py index 294e57f6f..f97434c0a 100755 --- a/helpers/bumper.py +++ b/helpers/bumper.py @@ -7,11 +7,11 @@ os.chdir(os.path.join(os.path.dirname(__file__), '..')) -chart_version = '7.2.1-0' +chart_version = '7.3.0' versions = { 6: '6.8.1', - 7: '7.2.0', + 7: '7.3.0', } file_patterns = [ @@ -28,7 +28,7 @@ blacklist = re.compile(r".*127.0.0.1.*") for major, version in versions.iteritems(): - r = re.compile(r"^({0})\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$".format(major)) + r = re.compile(r"{0}\.[0-9]*\.[0-9]*-?[0-9]?".format(major)) for pattern in file_patterns: for f in glob.glob(pattern): print(f) diff --git a/helpers/examples.mk b/helpers/examples.mk index ca42dde14..c874cd45f 100644 --- a/helpers/examples.mk +++ b/helpers/examples.mk @@ -1,7 +1,7 @@ GOSS_VERSION := v0.3.6 GOSS_FILE ?= goss.yaml GOSS_SELECTOR ?= release=$(RELEASE) -STACK_VERSION := 7.2.0 +STACK_VERSION := 7.3.0 goss: GOSS_CONTAINER=$$(kubectl get --no-headers=true pods -l $(GOSS_SELECTOR) -o custom-columns=:metadata.name | sed -n 1p ) && \ diff --git a/kibana/Chart.yaml b/kibana/Chart.yaml index 09a985f33..c16bd2c89 100755 --- a/kibana/Chart.yaml +++ b/kibana/Chart.yaml @@ -4,8 +4,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: kibana -version: 7.2.1-0 -appVersion: 7.2.0 +version: 7.3.0 +appVersion: 7.3.0 sources: - https://github.com/elastic/kibana icon: https://helm.elastic.co/icons/kibana.png diff --git a/kibana/README.md b/kibana/README.md index 87bb6025a..4362afc49 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -26,14 +26,14 @@ This chart is tested with the latest supported versions. The currently tested ve | 6.x | 7.x | | ----- | ----- | -| 6.8.1 | 7.2.0 | +| 6.8.1 | 7.3.0 | Examples of installing older major versions can be found in the [examples](./examples) directory. -While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.2.0` of Kibana it would look like this: +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.3.0` of Kibana it would look like this: ``` -helm install --name kibana elastic/kibana --set imageTag=7.2.0 +helm install --name kibana elastic/kibana --set imageTag=7.3.0 ``` ## Configuration @@ -46,7 +46,7 @@ helm install --name kibana elastic/kibana --set imageTag=7.2.0 | `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | | `secretMounts` | Allows you easily mount a secret as a file inside the deployment. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | | `image` | The Kibana docker image | `docker.elastic.co/kibana/kibana` | -| `imageTag` | The Kibana docker image tag | `7.2.0` | +| `imageTag` | The Kibana docker image tag | `7.3.0` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Kibana pods | `{}` | | `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 2Gi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | @@ -69,7 +69,7 @@ helm install --name kibana elastic/kibana --set imageTag=7.2.0 | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | | `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Kibana service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | | `service` | Configurable [service](https://kubernetes.io/docs/concepts/services-networking/service/) to expose the Kibana service. See [`values.yaml`](./values.yaml) for an example | `type: ClusterIP`
`port: 5601`
`nodePort:`
`annotations: {}` | -| `labels` | Configurable [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) applied to all Kibana pods | `{}` | +| `labels` | Configurable [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) applied to all Kibana pods | `{}` | ## Examples diff --git a/kibana/examples/default/test/goss.yaml b/kibana/examples/default/test/goss.yaml index cd40d36cd..3691f18c7 100644 --- a/kibana/examples/default/test/goss.yaml +++ b/kibana/examples/default/test/goss.yaml @@ -3,7 +3,7 @@ http: status: 200 timeout: 2000 body: - - '"number":"7.2.0"' + - '"number":"7.3.0"' http://localhost:5601/app/kibana: status: 200 diff --git a/kibana/values.yaml b/kibana/values.yaml index e253b3e66..cef949e62 100755 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -22,7 +22,7 @@ secretMounts: [] # subPath: kibana.keystore # optional image: "docker.elastic.co/kibana/kibana" -imageTag: "7.2.0" +imageTag: "7.3.0" imagePullPolicy: "IfNotPresent" # additionals labels diff --git a/metricbeat/Chart.yaml b/metricbeat/Chart.yaml index ba5db384f..827245d11 100755 --- a/metricbeat/Chart.yaml +++ b/metricbeat/Chart.yaml @@ -4,8 +4,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: metricbeat -version: 7.2.1-0 -appVersion: 7.2.0 +version: 7.3.0 +appVersion: 7.3.0 sources: - https://github.com/elastic/beats icon: https://helm.elastic.co/icons/metricbeat.png diff --git a/metricbeat/README.md b/metricbeat/README.md index 5f2dd02bd..81b60827a 100644 --- a/metricbeat/README.md +++ b/metricbeat/README.md @@ -26,14 +26,14 @@ This chart is tested with the latest supported versions. The currently tested ve | 6.x | 7.x | | ----- | ----- | -| 6.8.1 | 7.2.0 | +| 6.8.1 | 7.3.0 | Examples of installing older major versions can be found in the [examples](./examples) directory. -While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.2.0` of metricbeat it would look like this: +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.3.0` of metricbeat it would look like this: ``` -helm install --name metricbeat elastic/metricbeat --set imageTag=7.2.0 +helm install --name metricbeat elastic/metricbeat --set imageTag=7.3.0 ``` @@ -46,7 +46,7 @@ helm install --name metricbeat elastic/metricbeat --set imageTag=7.2.0 | `extraVolumes` | Any extra volumes to define for the pod | `[]` | | `hostPathRoot` | Fully-qualified [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) that will be used to persist Metricbeat registry data | `/var/lib` | | `image` | The Metricbeat docker image | `docker.elastic.co/beats/metricbeat` | -| `imageTag` | The Metricbeat docker image tag | `7.2.0` | +| `imageTag` | The Metricbeat docker image tag | `7.3.0` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | | `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | | `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles. | `true` | diff --git a/metricbeat/examples/default/test/goss-metrics.yaml b/metricbeat/examples/default/test/goss-metrics.yaml index ec57e8bee..4cafce058 100644 --- a/metricbeat/examples/default/test/goss-metrics.yaml +++ b/metricbeat/examples/default/test/goss-metrics.yaml @@ -21,12 +21,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.2.0' + - 'metricbeat-7.3.0' http://elasticsearch-master:9200/_search?q=metricset.name:state_deployment: status: 200 timeout: 2000 body: - - 'metricbeat-7.2.0' + - 'metricbeat-7.3.0' file: /usr/share/metricbeat/metricbeat.yml: @@ -41,4 +41,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://elasticsearch-master:9200' - - 'version: 7.2.0' + - 'version: 7.3.0' diff --git a/metricbeat/examples/default/test/goss.yaml b/metricbeat/examples/default/test/goss.yaml index edc9fdd2f..5184c44f9 100644 --- a/metricbeat/examples/default/test/goss.yaml +++ b/metricbeat/examples/default/test/goss.yaml @@ -29,12 +29,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.2.0' + - 'metricbeat-7.3.0' http://elasticsearch-master:9200/_search?q=metricset.name:container: status: 200 timeout: 2000 body: - - 'metricbeat-7.2.0' + - 'metricbeat-7.3.0' file: /usr/share/metricbeat/metricbeat.yml: @@ -49,4 +49,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://elasticsearch-master:9200' - - 'version: 7.2.0' + - 'version: 7.3.0' diff --git a/metricbeat/examples/oss/test/goss-metrics.yaml b/metricbeat/examples/oss/test/goss-metrics.yaml index 4b115200d..d8d930db9 100644 --- a/metricbeat/examples/oss/test/goss-metrics.yaml +++ b/metricbeat/examples/oss/test/goss-metrics.yaml @@ -21,12 +21,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.2.0' + - 'metricbeat-7.3.0' http://oss-master:9200/_search?q=metricset.name:state_deployment: status: 200 timeout: 2000 body: - - 'metricbeat-7.2.0' + - 'metricbeat-7.3.0' file: /usr/share/metricbeat/metricbeat.yml: @@ -40,4 +40,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://oss-master:9200' - - 'version: 7.2.0' + - 'version: 7.3.0' diff --git a/metricbeat/examples/oss/test/goss.yaml b/metricbeat/examples/oss/test/goss.yaml index e7788dd04..37ff2a76a 100644 --- a/metricbeat/examples/oss/test/goss.yaml +++ b/metricbeat/examples/oss/test/goss.yaml @@ -29,12 +29,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.2.0' + - 'metricbeat-7.3.0' http://oss-master:9200/_search?q=metricset.name:container: status: 200 timeout: 2000 body: - - 'metricbeat-7.2.0' + - 'metricbeat-7.3.0' file: /usr/share/metricbeat/metricbeat.yml: @@ -48,4 +48,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://oss-master:9200' - - 'version: 7.2.0' + - 'version: 7.3.0' diff --git a/metricbeat/examples/security/test/goss-metrics.yaml b/metricbeat/examples/security/test/goss-metrics.yaml index 86434f43a..78b91122d 100644 --- a/metricbeat/examples/security/test/goss-metrics.yaml +++ b/metricbeat/examples/security/test/goss-metrics.yaml @@ -21,7 +21,7 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.2.0' + - 'metricbeat-7.3.0' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' @@ -29,7 +29,7 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.2.0' + - 'metricbeat-7.3.0' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' @@ -46,4 +46,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: https://security-master:9200' - - 'version: 7.2.0' + - 'version: 7.3.0' diff --git a/metricbeat/examples/security/test/goss.yaml b/metricbeat/examples/security/test/goss.yaml index 05d6d0a87..eeb6eccff 100644 --- a/metricbeat/examples/security/test/goss.yaml +++ b/metricbeat/examples/security/test/goss.yaml @@ -29,7 +29,7 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.2.0' + - 'metricbeat-7.3.0' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' @@ -37,7 +37,7 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.2.0' + - 'metricbeat-7.3.0' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' @@ -54,4 +54,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: https://security-master:9200' - - 'version: 7.2.0' + - 'version: 7.3.0' diff --git a/metricbeat/values.yaml b/metricbeat/values.yaml index 7cae07ef4..29996d3f4 100755 --- a/metricbeat/values.yaml +++ b/metricbeat/values.yaml @@ -85,7 +85,7 @@ extraVolumes: [] hostPathRoot: /var/lib image: "docker.elastic.co/beats/metricbeat" -imageTag: "7.2.0" +imageTag: "7.3.0" imagePullPolicy: "IfNotPresent" imagePullSecrets: [] From 932f62b93e2aa2903234f89a9cedfbaba2028533 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Wed, 31 Jul 2019 17:12:24 +0200 Subject: [PATCH 103/115] [meta] Add dependency download to release script This is needed for metricbeat to download the child charts --- helpers/release.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helpers/release.py b/helpers/release.py index 811bde945..145995ff1 100644 --- a/helpers/release.py +++ b/helpers/release.py @@ -30,6 +30,9 @@ def run(cmd): for filepath in glob.iglob('*/Chart.yaml'): chart = os.path.split(os.path.dirname(filepath))[-1] + # Download dependencies + run(['helm', 'dependency', 'update', chart]) + # Package up the chart run(['helm', 'package', chart, '--destination', chart]) From 007d6e4096c4419200b7c52c56781e9e1974c369 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Thu, 1 Aug 2019 11:03:07 +0200 Subject: [PATCH 104/115] Update documentation and defaults for tmpl values Closes: #235 --- elasticsearch/README.md | 6 +++--- elasticsearch/values.yaml | 6 +++--- filebeat/README.md | 4 ++-- filebeat/values.yaml | 4 ++-- metricbeat/README.md | 4 ++-- metricbeat/values.yaml | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 924800a48..12b35fafe 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -66,9 +66,9 @@ helm install --name elasticsearch elastic/elasticsearch --set imageTag=7.3.0 | `esMajorVersion` | Used to set major version specific configuration. If you are using a custom image and not running the default Elasticsearch version you will need to set this to the version you are running (e.g. `esMajorVersion: 6`) | `""` | | `esConfig` | Allows you to add any config files in `/usr/share/elasticsearch/config/` such as `elasticsearch.yml` and `log4j2.properties`. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | | `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | -| `extraVolumes` | Additional volumes to be passed to the `tpl` function | | -| `extraVolumeMounts` | Additional volumeMounts to be passed to the `tpl` function | | -| `extraInitContainers` | Additional init containers to be passed to the `tpl` function | | +| `extraVolumes` | Templatable string of additional volumes to be passed to the `tpl` function | `""` | +| `extraVolumeMounts` | Templatable string of additional volumeMounts to be passed to the `tpl` function | `""` | +| `extraInitContainers` | Templatable string of additional init containers to be passed to the `tpl` function | `""` | | `secretMounts` | Allows you easily mount a secret as a file inside the statefulset. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | | `image` | The Elasticsearch docker image | `docker.elastic.co/elasticsearch/elasticsearch` | | `imageTag` | The Elasticsearch docker image tag | `7.3.0` | diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 13ca0626d..eaf9b3a99 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -90,16 +90,16 @@ persistence: enabled: true annotations: {} -extraVolumes: [] +extraVolumes: "" # - name: extras # emptyDir: {} -extraVolumeMounts: [] +extraVolumeMounts: "" # - name: extras # mountPath: /usr/share/extras # readOnly: true -extraInitContainers: [] +extraInitContainers: "" # - name: do-something # image: busybox # command: ['do', 'something'] diff --git a/filebeat/README.md b/filebeat/README.md index 7a6b3ec32..cbfe93605 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -46,8 +46,8 @@ helm install --name filebeat elastic/filebeat --set imageTag=7.3.0 | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | | `filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml`. See [values.yaml](./values.yaml) for an example of the formatting with the default configuration. | see [values.yaml](./values.yaml) | | `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | -| `extraVolumeMounts` | Any extra volumes mounts to define for the Filebeat container | `[]` | -| `extraVolumes` | Any extra volumes to define for the pod | `[]` | +| `extraVolumeMounts` | Templatable string of additional volumeMounts to be passed to the `tpl` function | `""` | +| `extraVolumes` | Templatable string of additional volumes to be passed to the `tpl` function | `""` | | `hostPathRoot` | Fully-qualified [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) that will be used to persist Filebeat registry data | `/var/lib` | | `image` | The Filebeat docker image | `docker.elastic.co/beats/filebeat` | | `imageTag` | The Filebeat docker image tag | `7.3.0` | diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 3da889cdd..f4fc22330 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -21,12 +21,12 @@ extraEnvs: [] # - name: MY_ENVIRONMENT_VAR # value: the_value_goes_here -extraVolumeMounts: [] +extraVolumeMounts: "" # - name: extras # mountPath: /usr/share/extras # readOnly: true -extraVolumes: [] +extraVolumes: "" # - name: extras # emptyDir: {} diff --git a/metricbeat/README.md b/metricbeat/README.md index 81b60827a..e48443f36 100644 --- a/metricbeat/README.md +++ b/metricbeat/README.md @@ -42,8 +42,8 @@ helm install --name metricbeat elastic/metricbeat --set imageTag=7.3.0 | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | | `metricbeatConfig` | Allows you to add any config files in `/usr/share/metricbeat` such as `metricbeat.yml`. See [values.yaml](./values.yaml) for an example of the formatting with the default configuration. | see [values.yaml](./values.yaml) | | `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | -| `extraVolumeMounts` | Any extra volumes mounts to define for the Metricbeat container | `[]` | -| `extraVolumes` | Any extra volumes to define for the pod | `[]` | +| `extraVolumeMounts` | Templatable string of additional volumeMounts to be passed to the `tpl` function | `""` | +| `extraVolumes` | Templatable string of additional volumes to be passed to the `tpl` function | `""` | | `hostPathRoot` | Fully-qualified [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) that will be used to persist Metricbeat registry data | `/var/lib` | | `image` | The Metricbeat docker image | `docker.elastic.co/beats/metricbeat` | | `imageTag` | The Metricbeat docker image tag | `7.3.0` | diff --git a/metricbeat/values.yaml b/metricbeat/values.yaml index 29996d3f4..3e628379c 100755 --- a/metricbeat/values.yaml +++ b/metricbeat/values.yaml @@ -72,12 +72,12 @@ extraEnvs: [] # - name: MY_ENVIRONMENT_VAR # value: the_value_goes_here -extraVolumeMounts: [] +extraVolumeMounts: "" # - name: extras # mountPath: /usr/share/extras # readOnly: true -extraVolumes: [] +extraVolumes: "" # - name: extras # emptyDir: {} From 1bc39378b380792961c319bd85dadbd9e2ae0f94 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Thu, 1 Aug 2019 14:53:27 +0200 Subject: [PATCH 105/115] [filebeat][metricbeat] Add configurable nodeSelector and affinity spec Fixes: #242 --- filebeat/README.md | 2 ++ filebeat/templates/daemonset.yaml | 9 +++++++-- filebeat/tests/filebeat_test.py | 28 ++++++++++++++++++++++++++++ filebeat/values.yaml | 4 ++++ metricbeat/README.md | 2 ++ metricbeat/templates/daemonset.yaml | 9 +++++++-- metricbeat/tests/metricbeat_test.py | 28 ++++++++++++++++++++++++++++ metricbeat/values.yaml | 4 ++++ 8 files changed, 82 insertions(+), 4 deletions(-) diff --git a/filebeat/README.md b/filebeat/README.md index 7a6b3ec32..eec7a9c60 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -64,6 +64,8 @@ helm install --name filebeat elastic/filebeat --set imageTag=7.3.0 | `secretMounts` | Allows you easily mount a secret as a file inside the `DaemonSet`. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | | `terminationGracePeriod` | Termination period (in seconds) to wait before killing Filebeat pod process on pod shutdown | `30` | | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | +| `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) | `{}` | +| `affinity` | Configurable [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) | `{}` | | `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) for the `DaemonSet`. By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually. | `RollingUpdate` | ## Examples diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index 69953df30..f6e705d27 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -36,8 +36,13 @@ spec: release: {{ .Release.Name | quote }} spec: {{- with .Values.tolerations }} - tolerations: -{{ toYaml . | indent 6 }} + tolerations: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 -}} {{- end }} serviceAccountName: {{ template "serviceAccount" . }} terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }} diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 00517e9c6..882603208 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -188,3 +188,31 @@ def test_adding_pod_labels(): ''' r = helm_template(config) assert r['daemonset'][name]['metadata']['labels']['app.kubernetes.io/name'] == 'filebeat' + + +def test_adding_a_node_selector(): + config = ''' +nodeSelector: + disktype: ssd +''' + r = helm_template(config) + assert r['daemonset'][name]['spec']['template']['spec']['nodeSelector']['disktype'] == 'ssd' + + +def test_adding_an_affinity_rule(): + config = ''' +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - filebeat + topologyKey: kubernetes.io/hostname +''' + + r = helm_template(config) + assert r['daemonset'][name]['spec']['template']['spec']['affinity']['podAntiAffinity'][ + 'requiredDuringSchedulingIgnoredDuringExecution'][0]['topologyKey'] == 'kubernetes.io/hostname' diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 3da889cdd..47a038947 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -90,6 +90,10 @@ terminationGracePeriod: 30 tolerations: [] +nodeSelector: {} + +affinity: {} + updateStrategy: RollingUpdate # Override various naming aspects of this chart diff --git a/metricbeat/README.md b/metricbeat/README.md index 81b60827a..b39ae62be 100644 --- a/metricbeat/README.md +++ b/metricbeat/README.md @@ -59,6 +59,8 @@ helm install --name metricbeat elastic/metricbeat --set imageTag=7.3.0 | `secretMounts` | Allows you easily mount a secret as a file inside the `DaemonSet`. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | | `terminationGracePeriod` | Termination period (in seconds) to wait before killing Metricbeat pod process on pod shutdown | `30` | | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | +| `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) | `{}` | +| `affinity` | Configurable [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) | `{}` | | `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) for the `DaemonSet`. By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually. | `RollingUpdate` | | `replicas` | The replica count for the metricbeat deployment talking to kube-state-metrics | `1` | diff --git a/metricbeat/templates/daemonset.yaml b/metricbeat/templates/daemonset.yaml index fd1b12f10..272c1fb01 100644 --- a/metricbeat/templates/daemonset.yaml +++ b/metricbeat/templates/daemonset.yaml @@ -33,8 +33,13 @@ spec: release: {{ .Release.Name | quote }} spec: {{- with .Values.tolerations }} - tolerations: -{{ toYaml . | indent 6 }} + tolerations: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 -}} {{- end }} serviceAccountName: {{ template "serviceAccount" . }} terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }} diff --git a/metricbeat/tests/metricbeat_test.py b/metricbeat/tests/metricbeat_test.py index d578fef6b..96fa7ef63 100644 --- a/metricbeat/tests/metricbeat_test.py +++ b/metricbeat/tests/metricbeat_test.py @@ -179,3 +179,31 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {'name': 'extras', 'emptyDir': {}} in extraVolume extraVolumeMounts = r['daemonset'][name]['spec']['template']['spec']['containers'][0]['volumeMounts'] assert {'name': 'extras', 'mountPath': '/usr/share/extras', 'readOnly': True} in extraVolumeMounts + + +def test_adding_a_node_selector(): + config = ''' +nodeSelector: + disktype: ssd +''' + r = helm_template(config) + assert r['daemonset'][name]['spec']['template']['spec']['nodeSelector']['disktype'] == 'ssd' + + +def test_adding_an_affinity_rule(): + config = ''' +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - metricbeat + topologyKey: kubernetes.io/hostname +''' + + r = helm_template(config) + assert r['daemonset'][name]['spec']['template']['spec']['affinity']['podAntiAffinity'][ + 'requiredDuringSchedulingIgnoredDuringExecution'][0]['topologyKey'] == 'kubernetes.io/hostname' diff --git a/metricbeat/values.yaml b/metricbeat/values.yaml index 29996d3f4..20bddd192 100755 --- a/metricbeat/values.yaml +++ b/metricbeat/values.yaml @@ -139,6 +139,10 @@ terminationGracePeriod: 30 tolerations: [] +nodeSelector: {} + +affinity: {} + updateStrategy: RollingUpdate # Override various naming aspects of this chart From b9e2b179bbda99d8f3b93f9e00247b6bd059e898 Mon Sep 17 00:00:00 2001 From: lancegerstner Date: Fri, 2 Aug 2019 11:25:44 -0500 Subject: [PATCH 106/115] Fixed indent on elasticsearch extraVolumes tpl. Was causing parsing errors --- elasticsearch/templates/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 30dcec51e..8a12eebaf 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -112,7 +112,7 @@ spec: name: {{ template "uname" . }}-config {{- end }} {{- if .Values.extraVolumes }} -{{ tpl .Values.extraVolumes . | indent 6 }} +{{ tpl .Values.extraVolumes . | indent 8 }} {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: From e8157f518b7e7b4273019540b7d43ec3455abc35 Mon Sep 17 00:00:00 2001 From: Trond Nordheim Date: Tue, 6 Aug 2019 12:44:51 +0200 Subject: [PATCH 107/115] Add priorityClassName to filebeat chart --- filebeat/README.md | 1 + filebeat/templates/daemonset.yaml | 3 +++ filebeat/tests/filebeat_test.py | 15 +++++++++++++++ filebeat/values.yaml | 4 ++++ 4 files changed, 23 insertions(+) diff --git a/filebeat/README.md b/filebeat/README.md index e1aa0faa9..96cf83dfe 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -66,6 +66,7 @@ helm install --name filebeat elastic/filebeat --set imageTag=7.3.0 | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | | `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) | `{}` | | `affinity` | Configurable [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) | `{}` | +| `priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `` | | `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) for the `DaemonSet`. By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually. | `RollingUpdate` | ## Examples diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index f6e705d27..ff0b9bc8b 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -41,6 +41,9 @@ spec: {{- with .Values.nodeSelector }} nodeSelector: {{ toYaml . | nindent 8 }} {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} {{- with .Values.affinity }} affinity: {{ toYaml . | nindent 8 -}} {{- end }} diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 882603208..5bd288921 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -216,3 +216,18 @@ def test_adding_an_affinity_rule(): r = helm_template(config) assert r['daemonset'][name]['spec']['template']['spec']['affinity']['podAntiAffinity'][ 'requiredDuringSchedulingIgnoredDuringExecution'][0]['topologyKey'] == 'kubernetes.io/hostname' + +def test_priority_class_name(): + config = ''' +priorityClassName: "" +''' + r = helm_template(config) + spec = r['daemonset'][name]['spec']['template']['spec'] + assert 'priorityClassName' not in spec + + config = ''' +priorityClassName: "highest" +''' + r = helm_template(config) + priority_class_name = r['daemonset'][name]['spec']['template']['spec']['priorityClassName'] + assert priority_class_name == "highest" diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 2a735e6b0..6f15685eb 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -94,6 +94,10 @@ nodeSelector: {} affinity: {} +# This is the PriorityClass settings as defined in +# https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass +priorityClassName: "" + updateStrategy: RollingUpdate # Override various naming aspects of this chart From 329f2ae7e4e67c1592129613c76c2b844b531ac2 Mon Sep 17 00:00:00 2001 From: Trond Nordheim Date: Wed, 7 Aug 2019 15:30:33 +0200 Subject: [PATCH 108/115] Clarify priorityClassName default for es chart --- elasticsearch/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 12b35fafe..55f32853c 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -83,7 +83,7 @@ helm install --name elasticsearch elastic/elasticsearch --set imageTag=7.3.0 | `volumeClaimTemplate` | Configuration for the [volumeClaimTemplate for statefulsets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-storage). You will want to adjust the storage (default `30Gi`) and the `storageClassName` if you are using a different storage class | `accessModes: [ "ReadWriteOnce" ]`
`resources.requests.storage: 30Gi` | | `persistence.annotations` | Additional persistence annotations for the `volumeClaimTemplate` | `{}` | | `persistence.enabled` | Enables a persistent volume for Elasticsearch data. Can be disabled for nodes that only have [roles](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) which don't require persistent data. | `true` | -| `priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `` | +| `priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `""` | | `antiAffinityTopologyKey` | The [anti-affinity topology key](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). By default this will prevent multiple Elasticsearch nodes from running on the same Kubernetes node | `kubernetes.io/hostname` | | `antiAffinity` | Setting this to hard enforces the [anti-affinity rules](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). If it is set to soft it will be done "best effort". Other values will be ignored. | `hard` | | `nodeAffinity` | Value for the [node affinity settings](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature) | `{}` | From 1a5a0219dffea32363c507266e311a0e7f8f0f92 Mon Sep 17 00:00:00 2001 From: Trond Nordheim Date: Wed, 7 Aug 2019 15:31:12 +0200 Subject: [PATCH 109/115] Clarify priorityClassName default for filebeat chart --- filebeat/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/README.md b/filebeat/README.md index 96cf83dfe..8ab66d260 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -66,7 +66,7 @@ helm install --name filebeat elastic/filebeat --set imageTag=7.3.0 | `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | | `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) | `{}` | | `affinity` | Configurable [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) | `{}` | -| `priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `` | +| `priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `""` | | `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) for the `DaemonSet`. By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually. | `RollingUpdate` | ## Examples From 123bdef93cce085ac9ff660db7f6b2f5a24706b7 Mon Sep 17 00:00:00 2001 From: Trond Nordheim Date: Wed, 7 Aug 2019 15:31:25 +0200 Subject: [PATCH 110/115] Clarify priorityClassName default for kibana chart --- kibana/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kibana/README.md b/kibana/README.md index 4362afc49..2fe63fd35 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -57,7 +57,7 @@ helm install --name kibana elastic/kibana --set imageTag=7.3.0 | `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000` | | `securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) for the container | `capabilities.drop:[ALL]`
`runAsNonRoot: true`
`runAsUser: 1000` | | `serviceAccount` | Allows you to overwrite the "default" [serviceAccount](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) for the pod | `[]` | -| `priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `` | +| `priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `""` | | `antiAffinityTopologyKey` | The [anti-affinity topology key](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). By default this will prevent multiple Kibana instances from running on the same Kubernetes node | `kubernetes.io/hostname` | | `antiAffinity` | Setting this to hard enforces the [anti-affinity rules](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). If it is set to soft it will be done "best effort" | `hard` | | `httpPort` | The http port that Kubernetes will use for the healthchecks and the service. | `5601` | From b466b62bb5d9b740b2dbfe72b75e8bb377bf24f9 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Fri, 9 Aug 2019 12:23:23 +0200 Subject: [PATCH 111/115] [metricbeat] Fix default configuration for kubernetes module This configuration didn't work at all since the nodes localhost is not reachable from the kubernetes pod. The test has also been fixed up to make sure that the data coming in actually has proper fields in it. The test was passing because the events were coming in but just with errors. --- metricbeat/examples/default/test/goss-metrics.yaml | 3 ++- metricbeat/examples/default/test/goss.yaml | 2 +- metricbeat/examples/security/values.yaml | 3 ++- metricbeat/templates/daemonset.yaml | 4 ++++ metricbeat/values.yaml | 3 ++- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/metricbeat/examples/default/test/goss-metrics.yaml b/metricbeat/examples/default/test/goss-metrics.yaml index 4cafce058..0e30826b1 100644 --- a/metricbeat/examples/default/test/goss-metrics.yaml +++ b/metricbeat/examples/default/test/goss-metrics.yaml @@ -22,7 +22,8 @@ http: timeout: 2000 body: - 'metricbeat-7.3.0' - http://elasticsearch-master:9200/_search?q=metricset.name:state_deployment: + + 'http://elasticsearch-master:9200/_search?q=metricset.name:state_container%20AND%20kubernetes.container.name:metricbeat': status: 200 timeout: 2000 body: diff --git a/metricbeat/examples/default/test/goss.yaml b/metricbeat/examples/default/test/goss.yaml index 5184c44f9..2306ee7d9 100644 --- a/metricbeat/examples/default/test/goss.yaml +++ b/metricbeat/examples/default/test/goss.yaml @@ -30,7 +30,7 @@ http: timeout: 2000 body: - 'metricbeat-7.3.0' - http://elasticsearch-master:9200/_search?q=metricset.name:container: + 'http://elasticsearch-master:9200/_search?q=metricset.name:container%20AND%20kubernetes.container.name:metricbeat': status: 200 timeout: 2000 body: diff --git a/metricbeat/examples/security/values.yaml b/metricbeat/examples/security/values.yaml index 5d3f92501..dfe90a10a 100644 --- a/metricbeat/examples/security/values.yaml +++ b/metricbeat/examples/security/values.yaml @@ -11,7 +11,8 @@ metricbeatConfig: - system - volume period: 10s - hosts: ["localhost:10255"] + host: "${NODE_NAME}" + hosts: ["${NODE_NAME}:10255"] processors: - add_kubernetes_metadata: in_cluster: true diff --git a/metricbeat/templates/daemonset.yaml b/metricbeat/templates/daemonset.yaml index 272c1fb01..a86850e50 100644 --- a/metricbeat/templates/daemonset.yaml +++ b/metricbeat/templates/daemonset.yaml @@ -105,6 +105,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName {{- if .Values.extraEnvs }} {{ toYaml .Values.extraEnvs | indent 8 }} {{- end }} diff --git a/metricbeat/values.yaml b/metricbeat/values.yaml index 7023436b3..c088a8793 100755 --- a/metricbeat/values.yaml +++ b/metricbeat/values.yaml @@ -14,7 +14,8 @@ metricbeatConfig: - system - volume period: 10s - hosts: ["localhost:10255"] + host: "${NODE_NAME}" + hosts: ["${NODE_NAME}:10255"] processors: - add_kubernetes_metadata: in_cluster: true From 8440172d39a67b0996a6b4881941fbc5eb94b758 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 13 Aug 2019 12:50:32 +0200 Subject: [PATCH 112/115] Fix octal literal to work in both Python 2 and Python 3 @Crazybus A repeat of #188 on a different file. --- metricbeat/tests/metricbeat_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/tests/metricbeat_test.py b/metricbeat/tests/metricbeat_test.py index 96fa7ef63..a274b6a90 100644 --- a/metricbeat/tests/metricbeat_test.py +++ b/metricbeat/tests/metricbeat_test.py @@ -136,7 +136,7 @@ def test_adding_in_metricbeat_config(): d = r['daemonset'][name]['spec']['template']['spec'] - assert {'configMap': {'name': name + '-config', 'defaultMode': 0600}, 'name': project + '-config'} in d['volumes'] + assert {'configMap': {'name': name + '-config', 'defaultMode': 0o600}, 'name': project + '-config'} in d['volumes'] assert {'mountPath': '/usr/share/metricbeat/metricbeat.yml', 'name': project + '-config', 'subPath': 'metricbeat.yml', 'readOnly': True} in d['containers'][0]['volumeMounts'] assert {'mountPath': '/usr/share/metricbeat/other-config.yml', 'name': project + '-config', 'subPath': 'other-config.yml', 'readOnly': True} in d['containers'][0]['volumeMounts'] From 8d8f6379a9c9e5602d46e751cc088ea7e749564d Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Thu, 22 Aug 2019 09:12:24 +0200 Subject: [PATCH 113/115] Bump google terraform provider to the latest The current version started failing a couple of days ago with some very weird terraform crashes. The latest version is working as expected though. https://devops-ci.elastic.co/job/elastic+helm-charts+pull-request+cluster-creation/129/KUBERNETES_VERSION=1.12,label=docker&&virtual/console --- helpers/terraform/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/terraform/main.tf b/helpers/terraform/main.tf index 5136b609e..292c881fa 100644 --- a/helpers/terraform/main.tf +++ b/helpers/terraform/main.tf @@ -1,7 +1,7 @@ provider "google" { project = "${var.project}" region = "${var.primary_region}" - version = "1.13.0" + version = "2.13.0" } terraform { From 7438b7ad23a8c2621866d2330e8d9419f89b8166 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 4 Jun 2019 10:19:21 +0200 Subject: [PATCH 114/115] [elasticsearch] Keystore integration Closes: #90 Adds a kubernetes native way to add strings and files to the Elasticsearch keystore. Previously you needed to manually create the keystore and upload it as a secret. There were a couple of issues with this approach. 1. The Elasticsearch keystore has an internal version for the format. If this is changed it meant needing to recreate each keystore again. 2. If you wanted to add a single new value it meant recreating the entire keystore again --- elasticsearch/README.md | 62 ++++++++-- elasticsearch/examples/config/Makefile | 19 ++++ elasticsearch/examples/config/README.md | 3 + elasticsearch/examples/config/test/goss.yaml | 26 +++++ elasticsearch/examples/config/values.yaml | 31 +++++ .../examples/config/watcher_encryption_key | 1 + elasticsearch/templates/statefulset.yaml | 45 ++++++++ elasticsearch/tests/elasticsearch_test.py | 106 ++++++++++++++++++ elasticsearch/values.yaml | 2 + helpers/matrix.yml | 1 + 10 files changed, 284 insertions(+), 12 deletions(-) create mode 100644 elasticsearch/examples/config/Makefile create mode 100644 elasticsearch/examples/config/README.md create mode 100644 elasticsearch/examples/config/test/goss.yaml create mode 100644 elasticsearch/examples/config/values.yaml create mode 100644 elasticsearch/examples/config/watcher_encryption_key diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 55f32853c..7ac117df6 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -111,6 +111,7 @@ helm install --name elasticsearch elastic/elasticsearch --set imageTag=7.3.0 | `schedulerName` | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods) | `nil` | | `masterTerminationFix` | A workaround needed for Elasticsearch < 7.2 to prevent master status being lost during restarts [#63](https://github.com/elastic/helm-charts/issues/63) | `false` | | `lifecycle` | Allows you to add lifecycle configuration. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | +| `keystore` | Allows you map Kubernetes secrets into the keystore. See the [config example](/elasticsearch/examples/config/values.yaml) and [how to use the keystore](#how-to-use-the-keystore) | `[]` | ## Try it out @@ -171,18 +172,55 @@ There are a couple reasons we recommend this. #### How to use the keystore? -1. Create a Kubernetes secret containing the [keystore](https://www.elastic.co/guide/en/elasticsearch/reference/current/secure-settings.html) - ``` - $ kubectl create secret generic elasticsearch-keystore --from-file=./elasticsearch.keystore - ``` -2. Mount it into the container via `secretMounts` - ``` - secretMounts: - - name: elasticsearch-keystore - secretName: elasticsearch-keystore - path: /usr/share/elasticsearch/config/elasticsearch.keystore - subPath: elasticsearch.keystore - ``` + +##### Basic example + +Create the secret, the key name needs to be the keystore key path. In this example we will create a secret from a file and from a literal string. + +``` +kubectl create secret generic encryption_key --from-file=xpack.watcher.encryption_key=./watcher_encryption_key +kubectl create secret generic slack_hook --from-literal=xpack.notification.slack.account.monitoring.secure_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' +``` + +To add these secrets to the keystore: +``` +keystore: + - secretName: encryption_key + - secretName: slack_hook +``` + +##### Multiple keys + +All keys in the secret will be added to the keystore. To create the previous example in one secret you could also do: + +``` +kubectl create secret generic keystore_secrets --from-file=xpack.watcher.encryption_key=./watcher_encryption_key --from-literal=xpack.notification.slack.account.monitoring.secure_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' +``` + +``` +keystore: + - secretName: keystore_secrets +``` + +##### Custom paths and keys + +If you are using these secrets for other applications (besides the Elasticsearch keystore) then it is also possible to specify the keystore path and which keys you want to add. Everything specified under each `keystore` item will be passed through to the `volumeMounts` section for [mounting the secret](https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets). In this example we will only add the `slack_hook` key from a secret that also has other keys. Our secret looks like this: + +``` +kubectl create secret generic slack_secrets --from-literal=slack_channel='#general' --from-literal=slack_hook='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' +``` + +We only want to add the `slack_hook` key to the keystore at path `xpack.notification.slack.account.monitoring.secure_url`. + +``` +keystore: + - secretName: slack_secrets + items: + - key: slack_hook + path: xpack.notification.slack.account.monitoring.secure_url +``` + +You can also take a look at the [config example](/elasticsearch/examples/config/) which is used as part of the automated testing pipeline. #### How to enable snapshotting? diff --git a/elasticsearch/examples/config/Makefile b/elasticsearch/examples/config/Makefile new file mode 100644 index 000000000..cf9c1f441 --- /dev/null +++ b/elasticsearch/examples/config/Makefile @@ -0,0 +1,19 @@ +default: test +include ../../../helpers/examples.mk + +RELEASE := helm-es-config + +install: + helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ ; \ + +secrets: + kubectl delete secret elastic-config-credentials elastic-config-secret elastic-config-slack elastic-config-custom-path || true + kubectl create secret generic elastic-config-credentials --from-literal=password=changeme --from-literal=username=elastic + kubectl create secret generic elastic-config-slack --from-literal=xpack.notification.slack.account.monitoring.secure_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' + kubectl create secret generic elastic-config-secret --from-file=xpack.watcher.encryption_key=./watcher_encryption_key + kubectl create secret generic elastic-config-custom-path --from-literal=slack_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' --from-literal=thing_i_don_tcare_about=test + +test: secrets install goss + +purge: + helm del --purge $(RELEASE) diff --git a/elasticsearch/examples/config/README.md b/elasticsearch/examples/config/README.md new file mode 100644 index 000000000..d98d836bf --- /dev/null +++ b/elasticsearch/examples/config/README.md @@ -0,0 +1,3 @@ +# Config + +An example testing suite for testing some of the optional features of this chart. diff --git a/elasticsearch/examples/config/test/goss.yaml b/elasticsearch/examples/config/test/goss.yaml new file mode 100644 index 000000000..848701370 --- /dev/null +++ b/elasticsearch/examples/config/test/goss.yaml @@ -0,0 +1,26 @@ +http: + http://localhost:9200/_cluster/health: + status: 200 + timeout: 2000 + body: + - 'green' + - '"number_of_nodes":1' + - '"number_of_data_nodes":1' + + http://localhost:9200: + status: 200 + timeout: 2000 + body: + - '"cluster_name" : "config"' + - '"name" : "config-master-0"' + - 'You Know, for Search' + +command: + "elasticsearch-keystore list": + exit-status: 0 + stdout: + - keystore.seed + - bootstrap.password + - xpack.notification.slack.account.monitoring.secure_url + - xpack.notification.slack.account.otheraccount.secure_url + - xpack.watcher.encryption_key diff --git a/elasticsearch/examples/config/values.yaml b/elasticsearch/examples/config/values.yaml new file mode 100644 index 000000000..ebde4f4d9 --- /dev/null +++ b/elasticsearch/examples/config/values.yaml @@ -0,0 +1,31 @@ +--- + +clusterName: "config" +replicas: 1 + +extraEnvs: + - name: ELASTIC_PASSWORD + valueFrom: + secretKeyRef: + name: elastic-credentials + key: password + - name: ELASTIC_USERNAME + valueFrom: + secretKeyRef: + name: elastic-credentials + key: username + +# This is just a dummy file to make sure that +# the keystore can be mounted at the same time +# as a custom elasticsearch.yml +esConfig: + elasticsearch.yml: | + path.data: /usr/share/elasticsearch/data + +keystore: + - secretName: elastic-config-secret + - secretName: elastic-config-slack + - secretName: elastic-config-custom-path + items: + - key: slack_url + path: xpack.notification.slack.account.otheraccount.secure_url diff --git a/elasticsearch/examples/config/watcher_encryption_key b/elasticsearch/examples/config/watcher_encryption_key new file mode 100644 index 000000000..b5f907866 --- /dev/null +++ b/elasticsearch/examples/config/watcher_encryption_key @@ -0,0 +1 @@ +supersecret diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 8a12eebaf..4eca980b0 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -111,6 +111,14 @@ spec: configMap: name: {{ template "uname" . }}-config {{- end }} +{{ if .Values.keystore }} + - name: keystore + emptyDir: {} +{{ end }} + {{- range .Values.keystore }} + - name: keystore-{{ .secretName }} + secret: {{ toYaml . | nindent 12 }} + {{- end }} {{- if .Values.extraVolumes }} {{ tpl .Values.extraVolumes . | indent 8 }} {{- end }} @@ -129,6 +137,38 @@ spec: resources: {{ toYaml .Values.initResources | indent 10 }} {{- end }} +{{ if .Values.keystore }} + - name: keystore + image: "{{ .Values.image }}:{{ .Values.imageTag }}" + command: + - sh + - -c + - | + #!/usr/bin/env bash + set -euo pipefail + + elasticsearch-keystore create + + for i in /tmp/keystoreSecrets/*/*; do + key=$(basename $i) + echo "Adding file $i to keystore key $key" + elasticsearch-keystore add-file "$key" "$i" + done + + # Add the bootstrap password since otherwise the Elasticsearch entrypoint tries to do this on startup + [ ! -z "$ELASTIC_PASSWORD" ] && echo $ELASTIC_PASSWORD | elasticsearch-keystore add -x bootstrap.password + + cp -a /usr/share/elasticsearch/config/elasticsearch.keystore /tmp/keystore/ + env: {{ toYaml .Values.extraEnvs | nindent 10 }} + resources: {{ toYaml .Values.initResources | nindent 10 }} + volumeMounts: + - name: keystore + mountPath: /tmp/keystore + {{- range .Values.keystore }} + - name: keystore-{{ .secretName }} + mountPath: /tmp/keystoreSecrets/{{ .secretName }} + {{- end }} +{{ end }} {{- if .Values.extraInitContainers }} {{ tpl .Values.extraInitContainers . | indent 6 }} {{- end }} @@ -219,6 +259,11 @@ spec: - name: "{{ template "uname" . }}" mountPath: /usr/share/elasticsearch/data {{- end }} +{{ if .Values.keystore }} + - name: keystore + mountPath: /usr/share/elasticsearch/config/elasticsearch.keystore + subPath: elasticsearch.keystore +{{ end }} {{- range .Values.secretMounts }} - name: {{ .name }} mountPath: {{ .path }} diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index cdc08a79a..a4949eef9 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -793,3 +793,109 @@ def test_adding_pod_labels(): ''' r = helm_template(config) assert r['statefulset'][uname]['metadata']['labels']['app.kubernetes.io/name'] == 'elasticsearch' + +def test_keystore_enable(): + config = '' + + r = helm_template(config) + s = r['statefulset'][uname]['spec']['template']['spec'] + + assert s['volumes'] == None + + config = ''' +keystore: + - secretName: test + ''' + + r = helm_template(config) + s = r['statefulset'][uname]['spec']['template']['spec'] + + assert {'name': 'keystore', 'emptyDir': {}} in s['volumes'] + +def test_keystore_init_container(): + config = '' + + r = helm_template(config) + i = r['statefulset'][uname]['spec']['template']['spec']['initContainers'][-1] + + assert i['name'] != 'keystore' + + config = ''' +keystore: + - secretName: test + ''' + + r = helm_template(config) + i = r['statefulset'][uname]['spec']['template']['spec']['initContainers'][-1] + + assert i['name'] == 'keystore' + +def test_keystore_mount(): + config = ''' +keystore: + - secretName: test +''' + + r = helm_template(config) + s = r['statefulset'][uname]['spec']['template']['spec'] + assert s['containers'][0]['volumeMounts'][-1] == { + 'mountPath': '/usr/share/elasticsearch/config/elasticsearch.keystore', + 'subPath': 'elasticsearch.keystore', + 'name': 'keystore' + } + +def test_keystore_init_volume_mounts(): + config = ''' +keystore: + - secretName: test + - secretName: test-with-custom-path + items: + - key: slack_url + path: xpack.notification.slack.account.otheraccount.secure_url +''' + r = helm_template(config) + s = r['statefulset'][uname]['spec']['template']['spec'] + assert s['initContainers'][-1]['volumeMounts'] == [ + { + 'mountPath': '/tmp/keystore', + 'name': 'keystore' + }, + { + 'mountPath': '/tmp/keystoreSecrets/test', + 'name': 'keystore-test' + }, + { + 'mountPath': '/tmp/keystoreSecrets/test-with-custom-path', + 'name': 'keystore-test-with-custom-path' + } + ] + +def test_keystore_volumes(): + config = ''' +keystore: + - secretName: test + - secretName: test-with-custom-path + items: + - key: slack_url + path: xpack.notification.slack.account.otheraccount.secure_url +''' + r = helm_template(config) + s = r['statefulset'][uname]['spec']['template']['spec'] + + assert { + 'name': 'keystore-test', + 'secret': { + 'secretName': 'test' + } + } in s['volumes'] + + assert { + 'name': 'keystore-test-with-custom-path', + 'secret': { + 'secretName': 'test-with-custom-path', + 'items': [{ + 'key': 'slack_url', + 'path': 'xpack.notification.slack.account.otheraccount.secure_url' + }] + } + } in s['volumes'] diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index eaf9b3a99..7ce13c314 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -210,3 +210,5 @@ lifecycle: {} sysctlInitContainer: enabled: true + +keystore: [] diff --git a/helpers/matrix.yml b/helpers/matrix.yml index d8dbf5396..221b35e47 100644 --- a/helpers/matrix.yml +++ b/helpers/matrix.yml @@ -5,6 +5,7 @@ CHART: - metricbeat ES_SUITE: - default + - config - multi - oss - security From d025a9bd6490e13801725e951f4887b2eb24feb2 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Thu, 22 Aug 2019 08:40:27 +0200 Subject: [PATCH 115/115] Properly quote bootstrap password --- elasticsearch/templates/statefulset.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 4eca980b0..fbf8e6577 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -111,14 +111,14 @@ spec: configMap: name: {{ template "uname" . }}-config {{- end }} -{{ if .Values.keystore }} +{{- if .Values.keystore }} - name: keystore emptyDir: {} -{{ end }} {{- range .Values.keystore }} - name: keystore-{{ .secretName }} secret: {{ toYaml . | nindent 12 }} {{- end }} +{{ end }} {{- if .Values.extraVolumes }} {{ tpl .Values.extraVolumes . | indent 8 }} {{- end }} @@ -156,7 +156,7 @@ spec: done # Add the bootstrap password since otherwise the Elasticsearch entrypoint tries to do this on startup - [ ! -z "$ELASTIC_PASSWORD" ] && echo $ELASTIC_PASSWORD | elasticsearch-keystore add -x bootstrap.password + [ ! -z "$ELASTIC_PASSWORD" ] && echo "$ELASTIC_PASSWORD" | elasticsearch-keystore add -x bootstrap.password cp -a /usr/share/elasticsearch/config/elasticsearch.keystore /tmp/keystore/ env: {{ toYaml .Values.extraEnvs | nindent 10 }}