Fix Alertmanager status page if clustering via gossip is disabled or sharding is enabled (#4184)

Signed-off-by: Xiaochao Dong (@damnever) <the.xcdong@gmail.com>

Author: damnever

CHANGELOG.md

@@ -28,6 +28,7 @@
 * [ENHANCEMENT] Added `tenant_ids` tag to tracing spans #4147
 * [BUGFIX] Purger: fix `Invalid null value in condition for column range` caused by `nil` value in range for WriteBatch query. #4128
 * [BUGFIX] Ingester: fixed infrequent panic caused by a race condition between TSDB mmap-ed head chunks truncation and queries. #4176
+* [BUGFIX] Alertmanager: fix Alertmanager status page if clustering via gossip is disabled or sharding is enabled. #4184
 * [BUGFIX] Ruler: fix `/ruler/rule_groups` endpoint doesn't work when used with object store. #4182
 * [BUGFIX] Ruler: Honor the evaluation delay for the `ALERTS` and `ALERTS_FOR_STATE` series. #4227
 * [BUGFIX] Fixed cache fetch error on Redis Cluster. #4056

integration/alertmanager_test.go

@@ -450,6 +450,14 @@ func TestAlertmanagerSharding(t *testing.T) {
 				}
 			}
 
+			// Endpoint: GET /multitenant_alertmanager/status
+			{
+				for _, c := range clients {
+					_, err := c.GetAlertmanagerStatusPage(context.Background())
+					assert.NoError(t, err)
+				}
+			}
+
 			// Endpoint: GET /status
 			{
 				for _, c := range clients {

integration/e2ecortex/client.go

@@ -27,20 +27,19 @@ import (
 	"github.com/cortexproject/cortex/pkg/ruler"
 )
 
-var (
-	ErrNotFound = errors.New("not found")
-)
+var ErrNotFound = errors.New("not found")
 
 // Client is a client used to interact with Cortex in integration tests
 type Client struct {
-	alertmanagerClient promapi.Client
-	querierAddress     string
-	rulerAddress       string
-	distributorAddress string
-	timeout            time.Duration
-	httpClient         *http.Client
-	querierClient      promv1.API
-	orgID              string
+	alertmanagerClient  promapi.Client
+	querierAddress      string
+	alertmanagerAddress string
+	rulerAddress        string
+	distributorAddress  string
+	timeout             time.Duration
+	httpClient          *http.Client
+	querierClient       promv1.API
+	orgID               string
 }
 
 // NewClient makes a new Cortex client
@@ -61,13 +60,14 @@ func NewClient(
 	}
 
 	c := &Client{
-		distributorAddress: distributorAddress,
-		querierAddress:     querierAddress,
-		rulerAddress:       rulerAddress,
-		timeout:            5 * time.Second,
-		httpClient:         &http.Client{},
-		querierClient:      promv1.NewAPI(querierAPIClient),
-		orgID:              orgID,
+		distributorAddress:  distributorAddress,
+		querierAddress:      querierAddress,
+		alertmanagerAddress: alertmanagerAddress,
+		rulerAddress:        rulerAddress,
+		timeout:             5 * time.Second,
+		httpClient:          &http.Client{},
+		querierClient:       promv1.NewAPI(querierAPIClient),
+		orgID:               orgID,
 	}
 
 	if alertmanagerAddress != "" {
@@ -391,6 +391,32 @@ type userConfig struct {
 	AlertmanagerConfig string            `yaml:"alertmanager_config"`
 }
 
+// GetAlertmanagerStatusPage gets the status page of alertmanager.
+func (c *Client) GetAlertmanagerStatusPage(ctx context.Context) ([]byte, error) {
+	return c.getRawPage(ctx, "http://"+c.alertmanagerAddress+"/multitenant_alertmanager/status")
+}
+
+func (c *Client) getRawPage(ctx context.Context, url string) ([]byte, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	content, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode/100 != 2 {
+		return nil, fmt.Errorf("fetching page failed with status %d and content %v", resp.StatusCode, string(content))
+	}
+	return content, nil
+}
+
 // GetAlertmanagerConfig gets the status of an alertmanager instance
 func (c *Client) GetAlertmanagerConfig(ctx context.Context) (*alertConfig.Config, error) {
 	u := c.alertmanagerClient.URL("/api/prom/api/v1/status", nil)
@@ -433,7 +459,6 @@ func (c *Client) SetAlertmanagerConfig(ctx context.Context, amConfig string, tem
 		AlertmanagerConfig: amConfig,
 		TemplateFiles:      templates,
 	})
-
 	if err != nil {
 		return err
 	}

pkg/alertmanager/alertmanager_http.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	statusPageTemplate = template.Must(template.New("main").Parse(`
+	ringStatusPageTemplate = template.Must(template.New("ringStatusPage").Parse(`
 	<!DOCTYPE html>
 	<html>
 		<head>
@@ -23,31 +23,90 @@ var (
 			<p>{{ .Message }}</p>
 		</body>
 	</html>`))
+
+	statusTemplate = template.Must(template.New("statusPage").Parse(`
+    <!doctype html>
+    <html>
+        <head><title>Cortex Alertmanager Status</title></head>
+        <body>
+            <h1>Cortex Alertmanager Status</h1>
+            {{ if not .ClusterInfo }}
+                <p>Alertmanager gossip-based clustering is disabled.</p>
+            {{ else }}
+                <h2>Node</h2>
+                <dl>
+                    <dt>Name</dt><dd>{{.ClusterInfo.self.Name}}</dd>
+                    <dt>Addr</dt><dd>{{.ClusterInfo.self.Addr}}</dd>
+                    <dt>Port</dt><dd>{{.ClusterInfo.self.Port}}</dd>
+                </dl>
+                <h3>Members</h3>
+                {{ with .ClusterInfo.members }}
+                <table>
+                    <tr><th>Name</th><th>Addr</th></tr>
+                    {{ range . }}
+                    <tr><td>{{ .Name }}</td><td>{{ .Addr }}</td></tr>
+                    {{ end }}
+                </table>
+                {{ else }}
+                <p>No peers</p>
+                {{ end }}
+            {{ end }}
+        </body>
+    </html>`))
 )
 
-func writeMessage(w http.ResponseWriter, message string) {
+func writeRingStatusMessage(w http.ResponseWriter, message string) {
 	w.WriteHeader(http.StatusOK)
-	err := statusPageTemplate.Execute(w, struct {
+	err := ringStatusPageTemplate.Execute(w, struct {
 		Message string
 	}{Message: message})
-
 	if err != nil {
 		level.Error(util_log.Logger).Log("msg", "unable to serve alertmanager ring page", "err", err)
 	}
 }
 
 func (am *MultitenantAlertmanager) RingHandler(w http.ResponseWriter, req *http.Request) {
 	if !am.cfg.ShardingEnabled {
-		writeMessage(w, "Alertmanager has no ring because sharding is disabled.")
+		writeRingStatusMessage(w, "Alertmanager has no ring because sharding is disabled.")
 		return
 	}
 
 	if am.State() != services.Running {
 		// we cannot read the ring before the alertmanager is in Running state,
 		// because that would lead to race condition.
-		writeMessage(w, "Alertmanager is not running yet.")
+		writeRingStatusMessage(w, "Alertmanager is not running yet.")
 		return
 	}
 
 	am.ring.ServeHTTP(w, req)
 }
+
+// GetStatusHandler returns the status handler for this multi-tenant
+// alertmanager.
+func (am *MultitenantAlertmanager) GetStatusHandler() StatusHandler {
+	return StatusHandler{
+		am: am,
+	}
+}
+
+// StatusHandler shows the status of the alertmanager.
+type StatusHandler struct {
+	am *MultitenantAlertmanager
+}
+
+// ServeHTTP serves the status of the alertmanager.
+func (s StatusHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	var clusterInfo map[string]interface{}
+	if s.am.peer != nil {
+		clusterInfo = s.am.peer.Info()
+	}
+	err := statusTemplate.Execute(w, struct {
+		ClusterInfo map[string]interface{}
+	}{
+		ClusterInfo: clusterInfo,
+	})
+	if err != nil {
+		level.Error(util_log.Logger).Log("msg", "unable to serve alertmanager status page", "err", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}

pkg/alertmanager/alertmanager_http_test.go

@@ -0,0 +1,83 @@
+package alertmanager
+
+import (
+	"context"
+	"io/ioutil"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/go-kit/kit/log"
+	"github.com/prometheus/alertmanager/cluster"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMultitenantAlertmanager_GetStatusHandler(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	var peer *cluster.Peer
+	{
+		logger := log.NewNopLogger()
+		createPeer := func(peers []string) (*cluster.Peer, error) {
+			return cluster.Create(
+				logger,
+				prometheus.NewRegistry(),
+				"127.0.0.1:0",
+				"",
+				peers,
+				true,
+				cluster.DefaultPushPullInterval,
+				cluster.DefaultGossipInterval,
+				cluster.DefaultTcpTimeout,
+				cluster.DefaultProbeTimeout,
+				cluster.DefaultProbeInterval,
+			)
+		}
+
+		peer1, err := createPeer(nil)
+		require.NoError(t, err)
+		require.NotNil(t, peer1)
+		err = peer1.Join(cluster.DefaultReconnectInterval, cluster.DefaultReconnectTimeout)
+		require.NoError(t, err)
+		go peer1.Settle(ctx, 0*time.Second)
+		require.NoError(t, peer1.WaitReady(ctx))
+		require.Equal(t, peer1.Status(), "ready")
+
+		peer2, err := createPeer([]string{peer1.Self().Address()})
+		require.NoError(t, err)
+		require.NotNil(t, peer2)
+		err = peer2.Join(cluster.DefaultReconnectInterval, cluster.DefaultReconnectTimeout)
+		require.NoError(t, err)
+		go peer2.Settle(ctx, 0*time.Second)
+		peer = peer2
+	}
+
+	for _, tt := range []struct {
+		am        *MultitenantAlertmanager
+		content   string
+		nocontent string
+	}{
+		{
+			am:        &MultitenantAlertmanager{peer: nil},
+			content:   "Alertmanager gossip-based clustering is disabled.",
+			nocontent: "Node",
+		},
+		{
+			am:        &MultitenantAlertmanager{peer: peer},
+			content:   "Members",
+			nocontent: "No peers",
+		},
+	} {
+		req := httptest.NewRequest("GET", "http://alertmanager.cortex/status", nil)
+		w := httptest.NewRecorder()
+		tt.am.GetStatusHandler().ServeHTTP(w, req)
+
+		resp := w.Result()
+		require.Equal(t, 200, w.Code)
+		body, _ := ioutil.ReadAll(resp.Body)
+		content := string(body)
+		require.Contains(t, content, tt.content)
+		require.NotContains(t, content, tt.nocontent)
+	}
+}