Fix readiness probe

pkg/controller/common/volume/downward_api.go

@@ -0,0 +1,46 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package volume
+
+import (
+	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume"
+	corev1 "k8s.io/api/core/v1"
+)
+
+var downwardApiVolume = corev1.Volume{
+	Name: volume.DownwardApiVolumeName,
+	VolumeSource: corev1.VolumeSource{
+		DownwardAPI: &corev1.DownwardAPIVolumeSource{
+			Items: []corev1.DownwardAPIVolumeFile{
+				{
+					Path: volume.LabelsFile,
+					FieldRef: &corev1.ObjectFieldSelector{
+						FieldPath: "metadata.labels",
+					},
+				},
+			},
+		},
+	},
+}
+
+var downwardApiVolumeMount = corev1.VolumeMount{
+	Name:      volume.DownwardApiVolumeName,
+	MountPath: volume.DownwardApiMountPath,
+	ReadOnly:  true,
+}
+
+type DownwardApi struct{}
+
+func (DownwardApi) Name() string {
+	return volume.DownwardApiVolumeName
+}
+
+func (DownwardApi) Volume() corev1.Volume {
+	return downwardApiVolume
+}
+
+func (DownwardApi) VolumeMount() corev1.VolumeMount {
+	return downwardApiVolumeMount
+}

pkg/controller/elasticsearch/nodespec/readiness_probe.go

@@ -28,7 +28,25 @@ func NewReadinessProbe() *corev1.Probe {
 
 const ReadinessProbeScriptConfigKey = "readiness-probe-script.sh"
 const ReadinessProbeScript = `#!/usr/bin/env bash
-# Consider a node to be healthy if it responds to a simple GET on "/_cat/nodes?local"
+
+# fail should be called as a last resort to help the user to understand why the probe failed
+function fail {
+  timestamp=$(date --iso-8601=seconds)
+  echo "{\"timestamp\": \"${timestamp}\", \"message\": \"readiness probe failed\", "$1"}" | tee /proc/1/fd/1
+  exit 1
+}
+
+labels="` + volume.DownwardApiMountPath + "/" + volume.LabelsFile + `"
+
+if [[ ! -f "${labels}" ]]; then
+  fail "\"reason\": \"${labels} does not exist\""
+fi
+
+# get Elasticsearch version from the downward API
+version=$(grep "elasticsearch.k8s.elastic.co/version" ${labels} | cut -d '=' -f 2)
+# remove quotes
+version=$(echo "${version}" | tr -d '"')
+
 CURL_TIMEOUT=${CURL_TIMEOUT:=3}
 
 # Check if PROBE_PASSWORD_PATH is set, otherwise fall back to its former name in 1.0.0.beta-1: PROBE_PASSWORD_FILE
@@ -46,14 +64,19 @@ else
   BASIC_AUTH=''
 fi
 
-# request Elasticsearch
-ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://127.0.0.1:9200/_cat/nodes?local"
+# request Elasticsearch on /
+ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://127.0.0.1:9200/"
 status=$(curl -o /dev/null -w "%{http_code}" --max-time $CURL_TIMEOUT -XGET -s -k ${BASIC_AUTH} $ENDPOINT)
+curl_rc=$?
+
+if [[ ${curl_rc} -ne 0 ]]; then
+  fail "\"curl_rc\": \"${curl_rc}\""
+fi
 
-# ready if status code 200
-if [[ $status == "200" ]]; then
-	exit 0
+# ready if status code 200, 503 is tolerable if ES version is 6.x
+if [[ ${status} == "200" ]] || [[ ${status} == "503" && ${version:0:2} == "6." ]]; then
+  exit 0
 else
-	exit 1
+  fail " \"status\": \"${status}\", \"version\":\"${version}\" "
 fi
 `

pkg/controller/elasticsearch/nodespec/volumes.go

@@ -17,6 +17,8 @@ import (
 	esvolume "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume"
 )
 
+var downwardApiVolume = volume.DownwardApi{}
+
 func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keystore.Resources) ([]corev1.Volume, []corev1.VolumeMount) {
 
 	configVolume := settings.ConfigSecretVolume(esv1.StatefulSet(esName, nodeSpec.Name))
@@ -70,6 +72,7 @@ func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keyst
 			httpCertificatesVolume.Volume(),
 			scriptsVolume.Volume(),
 			configVolume.Volume(),
+			downwardApiVolume.Volume(),
 		)...)
 	if keystoreResources != nil {
 		volumes = append(volumes, keystoreResources.Volume)
@@ -86,6 +89,7 @@ func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keyst
 		httpCertificatesVolume.VolumeMount(),
 		scriptsVolume.VolumeMount(),
 		configVolume.VolumeMount(),
+		downwardApiVolume.VolumeMount(),
 	)
 
 	return volumes, volumeMounts

pkg/controller/elasticsearch/volume/names.go

@@ -35,4 +35,8 @@ const (
 
 	ScriptsVolumeName      = "elastic-internal-scripts"
 	ScriptsVolumeMountPath = "/mnt/elastic-internal/scripts"
+
+	DownwardApiVolumeName = "downward-api"
+	DownwardApiMountPath  = "/mnt/elastic-internal/downward-api"
+	LabelsFile            = "labels"
 )