diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 40d6f94af8d0..516e68d27595 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/weaveworks/common/user" @@ -18,6 +19,8 @@ import ( "github.com/grafana/loki/pkg/logproto" ) +var readinessProbeSuccess = []byte("Ready") + var flushQueueLength = promauto.NewGauge(prometheus.GaugeOpts{ Name: "cortex_ingester_flush_queue_length", Help: "The total number of series pending in the flush queue.", @@ -195,13 +198,17 @@ func (*Ingester) Watch(*grpc_health_v1.HealthCheckRequest, grpc_health_v1.Health } // ReadinessHandler is used to indicate to k8s when the ingesters are ready for -// the addition removal of another ingester. Returns 204 when the ingester is +// the addition removal of another ingester. Returns 200 when the ingester is // ready, 500 otherwise. func (i *Ingester) ReadinessHandler(w http.ResponseWriter, r *http.Request) { - if err := i.lifecycler.CheckReady(r.Context()); err == nil { - w.WriteHeader(http.StatusNoContent) - } else { - w.WriteHeader(http.StatusInternalServerError) + if err := i.lifecycler.CheckReady(r.Context()); err != nil { + http.Error(w, "Not ready: "+err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + if _, err := w.Write(readinessProbeSuccess); err != nil { + level.Error(util.Logger).Log("msg", "error writing success message", "error", err) } } diff --git a/pkg/promtail/server/server.go b/pkg/promtail/server/server.go index 289ae4c18cf3..83242a88e4cf 100644 --- a/pkg/promtail/server/server.go +++ b/pkg/promtail/server/server.go @@ -10,6 +10,8 @@ import ( "strings" "text/template" + logutil "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/version" serverww "github.com/weaveworks/common/server" @@ -18,6 +20,11 @@ import ( "github.com/grafana/loki/pkg/promtail/targets" ) +var ( + readinessProbeFailure = "Not ready: Unable to find any logs to tail. Please verify permissions, volumes, scrape_config, etc." + readinessProbeSuccess = []byte("Ready") +) + // Server embed weaveworks server with static file and templating capability type Server struct { *serverww.Server @@ -155,10 +162,14 @@ func (s *Server) targets(rw http.ResponseWriter, _ *http.Request) { // ready serves the ready endpoint func (s *Server) ready(rw http.ResponseWriter, _ *http.Request) { - if s.tms.Ready() { - rw.WriteHeader(http.StatusNoContent) - } else { - rw.WriteHeader(http.StatusInternalServerError) + if !s.tms.Ready() { + http.Error(rw, readinessProbeFailure, http.StatusInternalServerError) + return + } + + rw.WriteHeader(http.StatusOK) + if _, err := rw.Write(readinessProbeSuccess); err != nil { + level.Error(logutil.Logger).Log("msg", "error writing success message", "error", err) } }