Add robustness to prom adapter http handler (istio#9010)

mixer/adapter/prometheus/prometheus.go

@@ -209,7 +209,14 @@ func (b *builder) Build(ctx context.Context, env adapter.Env) (adapter.Handler,
 		}
 	}
 
-	if err := b.srv.Start(env, promhttp.HandlerFor(b.registry, promhttp.HandlerOpts{})); err != nil {
+	// We want best-effort on metrics generation. It is important to log the failures, however,
+	// to help capture any breakages that may be hidden.
+	opts := promhttp.HandlerOpts{
+		ErrorHandling: promhttp.ContinueOnError,
+		ErrorLog:      &promLogger{logger: env.Logger()},
+	}
+
+	if err := b.srv.Start(env, promhttp.HandlerFor(b.registry, opts)); err != nil {
 		return nil, err
 	}
 
@@ -432,3 +439,11 @@ func computeSha(m proto.Marshaler, log adapter.Logger) [sha1.Size]byte {
 	}
 	return sha1.Sum(ba)
 }
+
+type promLogger struct {
+	logger adapter.Logger
+}
+
+func (pl *promLogger) Println(v ...interface{}) {
+	pl.logger.Errorf("Prometheus handler error: %s", fmt.Sprintln(v...)) // nolint: gas
+}

mixer/adapter/prometheus/prometheus_test.go

@@ -20,11 +20,13 @@ import (
 	"net/http"
 	"reflect"
 	"sort"
+	"strings"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	dto "github.com/prometheus/client_model/go"
 
 	"istio.io/istio/mixer/adapter/prometheus/config"
@@ -678,3 +680,23 @@ func makeConfigWithExpirationPolicy(policy *config.Params_MetricsExpirationPolic
 func expPolicy(expiry, check time.Duration) *config.Params_MetricsExpirationPolicy {
 	return &config.Params_MetricsExpirationPolicy{expiry, check}
 }
+
+func TestPromLogger_Println(t *testing.T) {
+	testEnvLogger := test.NewEnv(t)
+	var underTest promhttp.Logger
+	underTest = &promLogger{logger: testEnvLogger}
+
+	underTest.Println("Istio Service Mesh", 94.5, false)
+
+	logs := testEnvLogger.GetLogs()
+	if len(logs) != 1 {
+		t.Fatalf("Println() did not produce correct number of logs; got %d, want 1", len(logs))
+	}
+
+	line := logs[0]
+	want := "Prometheus handler error: Istio Service Mesh 94.5 false\n"
+	if !strings.EqualFold(line, want) {
+		t.Fatalf("Println() did not produce expected logs; got '%s', want '%s'", line, want)
+	}
+
+}