Adding initial infrestructure for metrics

go.mod

@@ -18,6 +18,8 @@ require (
 	github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0
 	github.com/pborman/uuid v1.2.1
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.11.1
+	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.8.4
 	github.com/testcontainers/testcontainers-go v0.19.0
 	gitlab.com/golang-commonmark/markdown v0.0.0-20211110145824-bf3e522c626a
@@ -30,9 +32,11 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cjlapao/common-go v0.0.39 // indirect
 	github.com/containerd/containerd v1.6.19 // indirect
 	github.com/cpuguy83/dockercfg v0.3.1 // indirect
@@ -75,6 +79,7 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/microsoft/kiota-authentication-azure-go v1.0.0 // indirect
 	github.com/microsoft/kiota-http-go v1.1.0 // indirect
 	github.com/microsoft/kiota-serialization-form-go v1.0.0 // indirect
@@ -100,10 +105,12 @@ require (
 	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.30.0 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rivo/uniseg v0.4.3 // indirect
 	github.com/rs/xid v1.4.0 // indirect
 	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/tinylib/msgp v1.1.8 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect

server/api.go

@@ -5,7 +5,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/mattermost/mattermost-plugin-msteams-sync/server/msteams"
@@ -29,6 +31,12 @@ func NewAPI(p *Plugin, store store.Store) *API {
 	router := mux.NewRouter()
 	api := &API{p: p, router: router, store: store}
 
+	enableMetrics := p.API.GetConfig().MetricsSettings.Enable
+	if enableMetrics != nil && *enableMetrics {
+		// set error counter middleware handler
+		router.Use(p.metricsMiddleware)
+	}
+
 	router.HandleFunc("/avatar/{userId:.*}", api.getAvatar).Methods("GET")
 	router.HandleFunc("/changes", api.processActivity).Methods("POST")
 	router.HandleFunc("/lifecycle", api.processLifecycle).Methods("POST")
@@ -145,6 +153,28 @@ func (a *API) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	a.router.ServeHTTP(w, r)
 }
 
+func (a *API) ServeHTTPWithMetrics(w http.ResponseWriter, r *http.Request) {
+	recorder := &StatusRecorder{
+		ResponseWriter: w,
+		Status:         200,
+	}
+	now := time.Now()
+
+	a.router.ServeHTTP(recorder, r)
+
+	elapsed := float64(time.Since(now)) / float64(time.Second)
+
+	var routeMatch mux.RouteMatch
+	a.router.Match(r, &routeMatch)
+	if routeMatch.Route != nil {
+		endpoint, err := routeMatch.Route.GetPathTemplate()
+		if err != nil {
+			endpoint = "unknown"
+		}
+		a.p.metricsService.ObserveAPIEndpointDuration(endpoint, r.Method, strconv.Itoa(recorder.Status), elapsed)
+	}
+}
+
 func (a *API) autocompleteTeams(w http.ResponseWriter, r *http.Request) {
 	out := []model.AutocompleteListItem{}
 	userID := r.Header.Get("Mattermost-User-ID")

server/metrics/metrics.go

@@ -0,0 +1,147 @@
+package metrics
+
+import (
+	"os"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+)
+
+const (
+	MetricsNamespace     = "msteams_connect"
+	MetricsSubsystemApp  = "app"
+	MetricsSubsystemHTTP = "http"
+	MetricsSubsystemAPI  = "api"
+
+	MetricsCloudInstallationLabel = "installationId"
+)
+
+type InstanceInfo struct {
+	InstallationID string
+}
+
+// Metrics used to instrumentate metrics in prometheus.
+type Metrics struct {
+	registry *prometheus.Registry
+
+	apiTime *prometheus.HistogramVec
+
+	httpRequestsTotal prometheus.Counter
+	httpErrorsTotal   prometheus.Counter
+
+	connectedUsersTotal prometheus.Gauge
+	syntheticUsersTotal prometheus.Gauge
+	linkedChannelsTotal prometheus.Gauge
+}
+
+// NewMetrics Factory method to create a new metrics collector.
+func NewMetrics(info InstanceInfo) *Metrics {
+	m := &Metrics{}
+
+	m.registry = prometheus.NewRegistry()
+	options := collectors.ProcessCollectorOpts{
+		Namespace: MetricsNamespace,
+	}
+	m.registry.MustRegister(collectors.NewProcessCollector(options))
+	m.registry.MustRegister(collectors.NewGoCollector())
+
+	additionalLabels := map[string]string{}
+	if info.InstallationID != "" {
+		additionalLabels[MetricsCloudInstallationLabel] = os.Getenv("MM_CLOUD_INSTALLATION_ID")
+	}
+
+	m.apiTime = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace:   MetricsNamespace,
+			Subsystem:   MetricsSubsystemAPI,
+			Name:        "time",
+			Help:        "Time to execute the api handler",
+			ConstLabels: additionalLabels,
+		},
+		[]string{"handler", "method", "status_code"},
+	)
+	m.registry.MustRegister(m.apiTime)
+
+	m.httpRequestsTotal = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemHTTP,
+		Name:        "requests_total",
+		Help:        "The total number of http API requests.",
+		ConstLabels: additionalLabels,
+	})
+	m.registry.MustRegister(m.httpRequestsTotal)
+
+	m.httpErrorsTotal = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemHTTP,
+		Name:        "errors_total",
+		Help:        "The total number of http API errors.",
+		ConstLabels: additionalLabels,
+	})
+	m.registry.MustRegister(m.httpErrorsTotal)
+
+	m.connectedUsersTotal = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemApp,
+		Name:        "connected_users_total",
+		Help:        "The total number of connected to MS Teams users.",
+		ConstLabels: additionalLabels,
+	})
+	m.registry.MustRegister(m.connectedUsersTotal)
+
+	m.syntheticUsersTotal = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemApp,
+		Name:        "synthetic_users_total",
+		Help:        "The total number of synthetic users.",
+		ConstLabels: additionalLabels,
+	})
+	m.registry.MustRegister(m.syntheticUsersTotal)
+
+	m.linkedChannelsTotal = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemApp,
+		Name:        "linked_channels_total",
+		Help:        "The total number of linked channels to MS Teams.",
+		ConstLabels: additionalLabels,
+	})
+	m.registry.MustRegister(m.linkedChannelsTotal)
+
+	return m
+}
+
+func (m *Metrics) ObserveAPIEndpointDuration(handler, method, statusCode string, elapsed float64) {
+	if m != nil {
+		m.apiTime.With(prometheus.Labels{"handler": handler, "method": method, "status_code": statusCode}).Observe(elapsed)
+	}
+}
+
+func (m *Metrics) ObserveConnectedUsersTotal(count int64) {
+	if m != nil {
+		m.connectedUsersTotal.Set(float64(count))
+	}
+}
+
+func (m *Metrics) ObserveSyntheticUsersTotal(count int64) {
+	if m != nil {
+		m.syntheticUsersTotal.Set(float64(count))
+	}
+}
+
+func (m *Metrics) ObserveLinkedChannelsTotal(count int64) {
+	if m != nil {
+		m.linkedChannelsTotal.Set(float64(count))
+	}
+}
+
+func (m *Metrics) IncrementHTTPRequests() {
+	if m != nil {
+		m.httpRequestsTotal.Inc()
+	}
+}
+
+func (m *Metrics) IncrementHTTPErrors() {
+	if m != nil {
+		m.httpErrorsTotal.Inc()
+	}
+}

server/metrics/service.go

@@ -0,0 +1,45 @@
+package metrics
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/sirupsen/logrus"
+)
+
+// Service prometheus to run the server.
+type Server struct {
+	*http.Server
+}
+
+type ErrorLoggerWrapper struct {
+}
+
+func (el *ErrorLoggerWrapper) Println(v ...interface{}) {
+	logrus.Warn("metric server error", v)
+}
+
+// NewMetricsServer factory method to create a new prometheus server.
+func NewMetricsServer(address string, metricsService *Metrics) *Server {
+	return &Server{
+		&http.Server{
+			ReadTimeout: 30 * time.Second,
+			Addr:        address,
+			Handler: promhttp.HandlerFor(metricsService.registry, promhttp.HandlerOpts{
+				ErrorLog: &ErrorLoggerWrapper{},
+			}),
+		},
+	}
+}
+
+// Run will start the prometheus server.
+func (h *Server) Run() error {
+	return errors.Wrap(h.Server.ListenAndServe(), "prometheus ListenAndServe")
+}
+
+// Shutdown will shutdown the prometheus server.
+func (h *Server) Shutdown() error {
+	return errors.Wrap(h.Server.Close(), "prometheus Close")
+}

server/middleware.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"net/http"
+)
+
+type StatusRecorder struct {
+	http.ResponseWriter
+	Status int
+}
+
+func (r *StatusRecorder) WriteHeader(status int) {
+	r.Status = status
+	r.ResponseWriter.WriteHeader(status)
+}
+
+func (p *Plugin) metricsMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if p.metricsService != nil {
+			p.metricsService.IncrementHTTPRequests()
+			recorder := &StatusRecorder{
+				ResponseWriter: w,
+				Status:         200,
+			}
+			next.ServeHTTP(recorder, r)
+			if recorder.Status < 200 || recorder.Status > 299 {
+				p.metricsService.IncrementHTTPErrors()
+			}
+		} else {
+			next.ServeHTTP(w, r)
+		}
+	})
+}

server/msteams/client.go

@@ -1119,6 +1119,9 @@ func (tc *ClientImpl) GetReply(teamID, channelID, messageID, replyID string) (*M
 }
 
 func (tc *ClientImpl) GetUserAvatar(userID string) ([]byte, error) {
+	if tc.client == nil {
+		return nil, errors.New("client not initialized")
+	}
 	photo, err := tc.client.Users().ByUserId(userID).Photo().Content().Get(tc.ctx, nil)
 	if err != nil {
 		return nil, NormalizeGraphAPIError(err)