[apm-server] allow customizing probes (#671)

This commit makes probes customizable using values.
The values where already partly existing but weren't used
in the deployment template.

elasticsearch/examples/security/Makefile

@@ -19,7 +19,7 @@ pull-elasticsearch-image:
 
 secrets:
 	docker rm -f elastic-helm-charts-certs || true
-	rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12 || true
+	rm -f elastic-certificates.p12 elastic-certificate.pem elastic-certificate.crt elastic-stack-ca.p12 || true
 	password=$$([ ! -z "$$ELASTIC_PASSWORD" ] && echo $$ELASTIC_PASSWORD || echo $$(docker run --rm busybox:1.31.1 /bin/sh -c "< /dev/urandom tr -cd '[:alnum:]' | head -c20")) && \
 	docker run --name elastic-helm-charts-certs -i -w /app \
 		$(ELASTICSEARCH_IMAGE) \
@@ -29,7 +29,9 @@ secrets:
 	docker cp elastic-helm-charts-certs:/app/elastic-certificates.p12 ./ && \
 	docker rm -f elastic-helm-charts-certs && \
 	openssl pkcs12 -nodes -passin pass:'' -in elastic-certificates.p12 -out elastic-certificate.pem && \
+	openssl x509 -outform der -in elastic-certificate.pem -out elastic-certificate.crt && \
 	kubectl create secret generic elastic-certificates --from-file=elastic-certificates.p12 && \
 	kubectl create secret generic elastic-certificate-pem --from-file=elastic-certificate.pem && \
+	kubectl create secret generic elastic-certificate-crt --from-file=elastic-certificate.crt && \
 	kubectl create secret generic elastic-credentials  --from-literal=password=$$password --from-literal=username=elastic && \
-	rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12
+	rm -f elastic-certificates.p12 elastic-certificate.pem elastic-certificate.crt elastic-stack-ca.p12

helpers/matrix.yml

@@ -26,6 +26,7 @@ LOGSTASH_SUITE:
   - default
   - oss
   - elasticsearch
+  - security
 APM_SERVER_SUITE:
   - default
   - oss

logstash/examples/security/Makefile

@@ -0,0 +1,14 @@
+default: test
+
+include ../../../helpers/examples.mk
+
+RELEASE := helm-logstash-security
+
+install:
+	helm upgrade --wait --timeout=900 --install $(RELEASE) --values values.yaml ../../
+
+test: install goss
+
+purge:
+	helm del --purge $(RELEASE)
+	kubectl delete $$(kubectl get pvc -l release=$(RELEASE) -o name)

logstash/examples/security/README.md

@@ -0,0 +1,28 @@
+# Security
+
+This example deploy Logstash 7.7.1 which connects to Elasticsearch using TLS
+(see [values][]).
+
+
+## Usage
+
+* Deploy [Elasticsearch Helm chart with security][].
+
+* Deploy Logstash chart: `make install`
+
+* You can now setup a port forward to query Logstash indices:
+
+  ```
+  kubectl port-forward svc/elasticsearch-master 9200
+  curl localhost:9200/_cat/indices
+  ```
+
+
+## Testing
+
+You can also run [goss integration tests][] using `make test`
+
+
+[elasticsearch helm chart with security]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/security/
+[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/logstash/examples/security/test/goss.yaml
+[values]: https://github.com/elastic/helm-charts/tree/master/logstash/examples/security/values.yaml

logstash/examples/security/test/goss.yaml

@@ -0,0 +1,64 @@
+mount:
+  /usr/share/logstash/data:
+    exists: true
+  /usr/share/logstash/config/logstash.yml:
+    exists: true
+    opts:
+      - ro
+  /usr/share/logstash/pipeline/uptime.conf:
+    exists: true
+    opts:
+      - ro
+
+user:
+  logstash:
+    exists: true
+    uid: 1000
+    gid: 1000
+
+http:
+  http://localhost:9600?pretty:
+    status: 200
+    timeout: 2000
+    body:
+      - '"host" : "helm-logstash-security-logstash-0"'
+      - '"version" : "7.7.1"'
+      - '"http_address" : "0.0.0.0:9600"'
+      - '"name" : "helm-logstash-security-logstash-0"'
+      - '"status" : "green"'
+      - '"workers" : 1'
+      - '"batch_size" : 125'
+      - '"batch_delay" : 50'
+  https://security-master:9200/_cat/indices:
+    status: 200
+    timeout: 2000
+    body:
+      - 'logstash'
+    allow-insecure: true
+    username: '{{ .Env.ELASTICSEARCH_USERNAME }}'
+    password: '{{ .Env.ELASTICSEARCH_PASSWORD }}'
+
+file:
+  /usr/share/logstash/config/logstash.yml:
+    exists: true
+    mode: "0644"
+    owner: root
+    group: logstash
+    filetype: file
+    contains:
+      - 'http.host: 0.0.0.0'
+      - 'xpack.monitoring.enabled: true'
+      - 'xpack.monitoring.elasticsearch.hosts: ["https://security-master:9200"]'
+      - 'xpack.monitoring.elasticsearch.ssl.certificate_authority: /usr/share/logstash/config/certs/elastic-certificate.crt'
+  /usr/share/logstash/pipeline/uptime.conf:
+    exists: true
+    mode: "0644"
+    owner: root
+    group: logstash
+    filetype: file
+    contains:
+      - 'input { exec { command => "uptime" interval => 30 } }'
+      - 'output { elasticsearch {'
+      - 'hosts => ["https://security-master:9200"]'
+      - 'cacert => "/usr/share/logstash/config/certs/elastic-certificate.crt"'
+      - 'index => "logstash"'

logstash/examples/security/values.yaml

@@ -0,0 +1,40 @@
+persistence:
+  enabled: true
+
+logstashConfig:
+  logstash.yml: |
+    http.host: 0.0.0.0
+    xpack.monitoring.enabled: true
+    xpack.monitoring.elasticsearch.username: '${ELASTICSEARCH_USERNAME}'
+    xpack.monitoring.elasticsearch.password: '${ELASTICSEARCH_PASSWORD}'
+    xpack.monitoring.elasticsearch.hosts: ["https://security-master:9200"]
+    xpack.monitoring.elasticsearch.ssl.certificate_authority: /usr/share/logstash/config/certs/elastic-certificate.crt
+
+logstashPipeline:
+  uptime.conf: |
+    input { exec { command => "uptime" interval => 30 } }
+    output { elasticsearch {
+      hosts => ["https://security-master:9200"]
+      cacert => "/usr/share/logstash/config/certs/elastic-certificate.crt"
+      user => '${ELASTICSEARCH_USERNAME}'
+      password => '${ELASTICSEARCH_PASSWORD}'
+      index => "logstash"
+      }
+    }
+
+secretMounts:
+  - name: elastic-certificate-crt
+    secretName: elastic-certificate-crt
+    path: /usr/share/logstash/config/certs
+
+extraEnvs:
+  - name: 'ELASTICSEARCH_USERNAME'
+    valueFrom:
+      secretKeyRef:
+        name: elastic-credentials
+        key: username
+  - name: 'ELASTICSEARCH_PASSWORD'
+    valueFrom:
+      secretKeyRef:
+        name: elastic-credentials
+        key: password