[elasticsearch] Keystore integration

Closes: #90

Adds a kubernetes native way to add strings and files to the
Elasticsearch keystore.

Previously you needed to manually create the keystore and upload
it as a secret. There were a couple of issues with this approach.

1. The Elasticsearch keystore has an internal version for the format. If
this is changed it meant needing to recreate each keystore again.

2. If you wanted to add a single new value it meant recreating the
entire keystore again

elasticsearch/README.md

@@ -111,6 +111,7 @@ helm install --name elasticsearch elastic/elasticsearch --set imageTag=7.3.0
 | `schedulerName`               | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods)                                                                                                                                                                          | `nil`                                                                                                                     |
 | `masterTerminationFix`        | A workaround needed for Elasticsearch < 7.2 to prevent master status being lost during restarts [#63](https://github.com/elastic/helm-charts/issues/63)                                                                                                                                                                    | `false`                                                                                                                   |
 | `lifecycle`                   | Allows you to add lifecycle configuration. See [values.yaml](./values.yaml) for an example of the formatting.                                                                                                                                                                                                              | `{}`                                                                                                                      |
+| `keystore`                 | Allows you map Kubernetes secrets into the keystore. See the [config example](/elasticsearch/examples/config/values.yaml) and [how to use the keystore](#how-to-use-the-keystore)                                                                                                                                             | `enabled: false`                                                                                                          |
 
 ## Try it out
 
@@ -171,18 +172,9 @@ There are a couple reasons we recommend this.
 
 #### How to use the keystore?
 
-1. Create a Kubernetes secret containing the [keystore](https://www.elastic.co/guide/en/elasticsearch/reference/current/secure-settings.html)
-   ```
-   $ kubectl create secret generic elasticsearch-keystore --from-file=./elasticsearch.keystore
-   ```
-2. Mount it into the container via `secretMounts`
-   ```
-   secretMounts:
-   - name: elasticsearch-keystore
-     secretName: elasticsearch-keystore
-     path: /usr/share/elasticsearch/config/elasticsearch.keystore
-     subPath: elasticsearch.keystore
-   ```
+Take a look a the [config example](/elasticsearch/examples/config/values.yaml) which has a tested version of adding strings and files to the keystore.
+
+If you have basic authentication enabled you will also need to set `keystore.strings.bootstrap.password: '${ELASTIC_PASSWORD}'`.
 
 #### How to enable snapshotting?
 

elasticsearch/examples/config/Makefile

@@ -0,0 +1,18 @@
+default: test
+include ../../../helpers/examples.mk
+
+RELEASE := helm-es-config
+
+install:
+	helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ ; \
+
+secrets:
+	kubectl delete secret elastic-config-credentials elastic-config-secret elastic-config-slack || true
+	kubectl create secret generic elastic-config-credentials --from-literal=password=changeme --from-literal=username=elastic
+	kubectl create secret generic elastic-config-slack --from-literal=slack_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd'
+	kubectl create secret generic elastic-config-secret --from-file=./watcher_encryption_key
+
+test: secrets install goss
+
+purge:
+	helm del --purge $(RELEASE)

elasticsearch/examples/config/README.md

@@ -0,0 +1,3 @@
+# Config
+
+An example testing suite for testing some of the optional features of this chart.

elasticsearch/examples/config/test/goss.yaml

@@ -0,0 +1,25 @@
+http:
+  http://localhost:9200/_cluster/health:
+    status: 200
+    timeout: 2000
+    body:
+      - 'green'
+      - '"number_of_nodes":1'
+      - '"number_of_data_nodes":1'
+
+  http://localhost:9200:
+    status: 200
+    timeout: 2000
+    body:
+      - '"cluster_name" : "config"'
+      - '"name" : "config-master-0"'
+      - 'You Know, for Search'
+
+command:
+  "elasticsearch-keystore list":
+    exit-status: 0
+    stdout:
+      - keystore.seed
+      - bootstrap.password
+      - xpack.notification.slack.account.monitoring.secure_url
+      - xpack.watcher.encryption_key

elasticsearch/examples/config/values.yaml

@@ -0,0 +1,38 @@
+---
+
+clusterName: "config"
+replicas: 1
+
+extraEnvs:
+  - name: ELASTIC_PASSWORD
+    valueFrom:
+      secretKeyRef:
+        name: elastic-credentials
+        key: password
+  - name: ELASTIC_USERNAME
+    valueFrom:
+      secretKeyRef:
+        name: elastic-credentials
+        key: username
+  - name: SLACK_URL
+    valueFrom:
+      secretKeyRef:
+        name: elastic-config-slack
+        key: slack_url
+
+esConfig:
+  elasticsearch.yml: |
+    path.data: /usr/share/elasticsearch/data
+
+secretMounts:
+  - name: elastic-config-secret
+    secretName: elastic-config-secret
+    path: /usr/share/elasticsearch/config/secret
+
+keystore:
+  enabled: true
+  strings:
+    bootstrap.password: '${ELASTIC_PASSWORD}'
+    xpack.notification.slack.account.monitoring.secure_url: '${SLACK_URL}'
+  files:
+    xpack.watcher.encryption_key: /usr/share/elasticsearch/config/secret/watcher_encryption_key

elasticsearch/examples/config/watcher_encryption_key

@@ -0,0 +1 @@
+supersecret

elasticsearch/templates/statefulset.yaml

@@ -111,6 +111,10 @@ spec:
           configMap:
             name: {{ template "uname" . }}-config
         {{- end }}
+{{ if .Values.keystore.enabled }}
+        - name: keystore
+          emptyDir: {}
+{{ end }}
       {{- if .Values.extraVolumes }}
 {{ tpl .Values.extraVolumes . | indent 6 }}
       {{- end }}
@@ -129,6 +133,44 @@ spec:
         resources:
 {{ toYaml .Values.initResources | indent 10 }}
       {{- end }}
+{{ if .Values.keystore.enabled }}
+      - name: keystore
+        image: "{{ .Values.image }}:{{ .Values.imageTag }}"
+        command:
+        - sh
+        - -c
+        - |
+          #!/usr/bin/env bash
+          set -euo pipefail
+          elasticsearch-keystore create
+          {{ range $path, $file := .Values.keystore.files -}}
+          echo 'Adding file "{{ $file }}" to keystore at path "{{ $path }}"'
+          elasticsearch-keystore add-file "{{ $path }}" "{{ $file }}"
+          {{- end -}}
+          {{ range $path, $string := .Values.keystore.strings }}
+          echo 'Adding string to path "{{ $path }}"'
+          echo "{{ $string }}" | elasticsearch-keystore add -x "{{ $path }}"
+          {{- end }}
+          cp -a /usr/share/elasticsearch/config/elasticsearch.keystore /tmp/keystore/
+{{- if .Values.extraEnvs }}
+        env:
+{{ toYaml .Values.extraEnvs | indent 10 }}
+{{- end }}
+        resources:
+{{ toYaml .Values.initResources | indent 10 }}
+        volumeMounts:
+{{ if .Values.keystore.enabled }}
+          - name: keystore
+            mountPath: /tmp/keystore
+{{ end }}
+          {{- range .Values.secretMounts }}
+          - name: {{ .name }}
+            mountPath: {{ .path }}
+            {{- if .subPath }}
+            subPath: {{ .subPath }}
+            {{- end }}
+          {{- end }}
+{{ end }}
       {{- if .Values.extraInitContainers }}
 {{ tpl .Values.extraInitContainers . | indent 6 }}
       {{- end }}
@@ -219,6 +261,11 @@ spec:
           - name: "{{ template "uname" . }}"
             mountPath: /usr/share/elasticsearch/data
           {{- end }}
+{{ if .Values.keystore.enabled }}
+          - name: keystore
+            mountPath: /usr/share/elasticsearch/config/elasticsearch.keystore
+            subPath: elasticsearch.keystore
+{{ end }}
           {{- range .Values.secretMounts }}
           - name: {{ .name }}
             mountPath: {{ .path }}

elasticsearch/tests/elasticsearch_test.py

@@ -793,3 +793,70 @@ def test_adding_pod_labels():
 '''
     r = helm_template(config)
     assert r['statefulset'][uname]['metadata']['labels']['app.kubernetes.io/name'] == 'elasticsearch'
+
+def test_keystore_enable():
+    config = ''
+
+    r = helm_template(config)
+    s = r['statefulset'][uname]['spec']['template']['spec']
+
+    assert s['volumes'] == None
+
+    config = '''
+keystore:
+  enabled: true
+    '''
+
+    r = helm_template(config)
+    s = r['statefulset'][uname]['spec']['template']['spec']
+
+    assert {'name': 'keystore', 'emptyDir': {}} in s['volumes']
+
+def test_keystore_init_container():
+    config = ''
+
+    r = helm_template(config)
+    i = r['statefulset'][uname]['spec']['template']['spec']['initContainers'][-1]
+
+    assert i['name'] != 'keystore'
+
+    config = '''
+keystore:
+  enabled: true
+    '''
+
+    r = helm_template(config)
+    i = r['statefulset'][uname]['spec']['template']['spec']['initContainers'][-1]
+
+    assert i['name'] == 'keystore'
+
+def test_keystore_mount():
+    config = '''
+keystore:
+  enabled: true
+'''
+
+    r = helm_template(config)
+    s = r['statefulset'][uname]['spec']['template']['spec']
+    assert s['containers'][0]['volumeMounts'][-1] == {
+        'mountPath': '/usr/share/elasticsearch/config/elasticsearch.keystore',
+        'subPath': 'elasticsearch.keystore',
+        'name': 'keystore'
+    }
+
+def test_keystore_bootstrap():
+    config = '''
+keystore:
+  enabled: true
+  strings:
+    bootstrap.password: '${ELASTIC_PASSWORD}'
+    xpack.notification.slack.account.monitoring.secure_url: "https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd"
+  files:
+    gcs.client.default.credentials_file: /usr/share/elasticsearch/config/gcs-credentials.json
+'''
+    r = helm_template(config)
+    i = r['statefulset'][uname]['spec']['template']['spec']['initContainers'][1]
+    command = " ".join(i['command'])
+    assert 'echo "${ELASTIC_PASSWORD}" | elasticsearch-keystore add -x "bootstrap.password"' in command
+    assert 'echo "https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd" | elasticsearch-keystore add -x "xpack.notification.slack.account.monitoring.secure_url"' in command
+    assert 'elasticsearch-keystore add-file "gcs.client.default.credentials_file" "/usr/share/elasticsearch/config/gcs-credentials.json"' in command

elasticsearch/values.yaml

@@ -210,3 +210,11 @@ lifecycle: {}
 
 sysctlInitContainer:
   enabled: true
+
+keystore:
+  enabled: false
+#  strings:
+#    bootstrap.password: '${ELASTIC_PASSWORD}'
+#    xpack.notification.slack.account.monitoring.secure_url: "hello world"
+#  files:
+#    gcs.client.default.credentials_file: /usr/share/elasticsearch/config/gcs-credentials.json

helpers/matrix.yml

@@ -5,6 +5,7 @@ CHART:
   - metricbeat
 ES_SUITE:
   - default
+  - config
   - multi
   - oss
   - security