QueryFrontend: Return server side performance information on server-timing header

CHANGELOG.md

@@ -7,6 +7,7 @@
 * [ENHANCEMENT] Prometheus upgraded. #3739
   * Avoid unnecessary `runtime.GC()` during compactions.
   * Prevent compaction loop in TSDB on data gap.
+* [ENHANCEMENT] Return server side performance metrics for query-frontend (using Server-timing header). #3685
 * [BUGFIX] HA Tracker: don't track as error in the `cortex_kv_request_duration_seconds` metric a CAS operation intentionally aborted. #3745
 
 ## 1.7.0 in progress

integration/query_frontend_test.go

@@ -302,6 +302,13 @@ func runQueryFrontendTest(t *testing.T, cfg queryFrontendTestConfig) {
 			assert.Equal(t, model.Time(1595846750806), matrix[0].Values[2].Timestamp)
 		}
 
+		// No need to repeat the test on Server-Timing header for each user.
+		if userID == 0 && cfg.queryStatsEnabled {
+			res, _, err := c.QueryRaw("{instance=~\"hello.*\"}")
+			require.NoError(t, err)
+			require.Regexp(t, "querier_wall_time;dur=[0-9.]*, response_time;dur=[0-9.]*$", res.Header.Values("Server-Timing")[0])
+		}
+
 		// In this test we do ensure that the /series start/end time is ignored and Cortex
 		// always returns series in ingesters memory. No need to repeat it for each user.
 		if userID == 0 {
@@ -332,6 +339,11 @@ func runQueryFrontendTest(t *testing.T, cfg queryFrontendTestConfig) {
 	if cfg.testMissingMetricName {
 		extra++
 	}
+
+	if cfg.queryStatsEnabled {
+		extra++
+	}
+
 	require.NoError(t, queryFrontend.WaitSumMetrics(e2e.Equals(numUsers*numQueriesPerUser+extra), "cortex_query_frontend_queries_total"))
 
 	// The number of received request is greater then the query requests because include

pkg/frontend/transport/handler.go

@@ -9,6 +9,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
@@ -27,6 +28,7 @@ import (
 const (
 	// StatusClientClosedRequest is the status code for when a client request cancellation of an http request
 	StatusClientClosedRequest = 499
+	ServiceTimingHeaderName   = "Server-Timing"
 )
 
 var (
@@ -114,6 +116,10 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		hs[h] = vs
 	}
 
+	if f.cfg.QueryStatsEnabled {
+		writeServiceTimingHeader(queryResponseTime, hs, stats)
+	}
+
 	w.WriteHeader(resp.StatusCode)
 	// we don't check for copy error as there is no much we can do at this point
 	_, _ = io.Copy(w, resp.Body)
@@ -201,3 +207,17 @@ func writeError(w http.ResponseWriter, err error) {
 	}
 	server.WriteError(w, err)
 }
+
+func writeServiceTimingHeader(queryResponseTime time.Duration, headers http.Header, stats *querier_stats.Stats) {
+	if stats != nil {
+		parts := make([]string, 0)
+		parts = append(parts, statsValue("querier_wall_time", stats.LoadWallTime()))
+		parts = append(parts, statsValue("response_time", queryResponseTime))
+		headers.Set(ServiceTimingHeaderName, strings.Join(parts, ", "))
+	}
+}
+
+func statsValue(name string, d time.Duration) string {
+	durationInMs := strconv.FormatFloat(float64(d)/float64(time.Millisecond), 'f', -1, 64)
+	return name + ";dur=" + durationInMs
+}