feat(pkger): extend metrics to collect remote sources

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 1. [18387](https://github.com/influxdata/influxdb/pull/18387): Integrate query cancellation after queries have been submitted
 1. [18515](https://github.com/influxdata/influxdb/pull/18515): Extend templates with the source file|url|reader.
+1. [18539](https://github.com/influxdata/influxdb/pull/18539): Collect stats on installed influxdata community template usage.
 
 ## v2.0.0-beta.12 [2020-06-12]
 

authorization/middleware_metrics.go

@@ -17,7 +17,7 @@ type AuthMetrics struct {
 
 var _ influxdb.AuthorizationService = (*AuthMetrics)(nil)
 
-func NewAuthMetrics(reg prometheus.Registerer, s influxdb.AuthorizationService, opts ...metric.MetricsOption) *AuthMetrics {
+func NewAuthMetrics(reg prometheus.Registerer, s influxdb.AuthorizationService, opts ...metric.ClientOptFn) *AuthMetrics {
 	o := metric.ApplyMetricOpts(opts...)
 	return &AuthMetrics{
 		rec:         metric.New(reg, o.ApplySuffix("token")),

http/swagger.yml

@@ -7650,7 +7650,7 @@ components:
           type: string
         stackID:
           type: string
-        package:
+        template:
           type: object
           properties:
             contentType:
@@ -7661,7 +7661,7 @@ components:
                 type: string
             package:
               $ref: "#/components/schemas/Pkg"
-        packages:
+        templates:
           type: array
           items:
             type: object

kit/metric/client.go

@@ -1,7 +1,6 @@
 package metric
 
 import (
-	"fmt"
 	"time"
 
 	"github.com/influxdata/influxdb/v2"
@@ -10,63 +9,143 @@ import (
 
 // REDClient is a metrics client for collection RED metrics.
 type REDClient struct {
-	// RED metrics
-	reqs *prometheus.CounterVec
-	errs *prometheus.CounterVec
-	durs *prometheus.HistogramVec
+	metrics []metricCollector
 }
 
 // New creates a new REDClient.
-func New(reg prometheus.Registerer, service string) *REDClient {
-	// MiddlewareMetrics is a metrics service middleware for the notification endpoint service.
-	const namespace = "service"
-
-	reqs := prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: namespace,
-		Subsystem: service,
-		Name:      "call_total",
-		Help:      fmt.Sprintf("Number of calls to the %s service", service),
-	}, []string{"method"})
-
-	errs := prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: namespace,
-		Subsystem: service,
-		Name:      "error_total",
-		Help:      fmt.Sprintf("Number of errors encountered when calling the %s service", service),
-	}, []string{"method", "code"})
-
-	durs := prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: namespace,
-		Subsystem: service,
-		Name:      "duration",
-		Help:      fmt.Sprintf("Duration of %s service calls", service),
-	}, []string{"method"})
-
-	reg.MustRegister(reqs, errs, durs)
-
-	return &REDClient{
-		reqs: reqs,
-		errs: errs,
-		durs: durs,
+func New(reg prometheus.Registerer, service string, opts ...ClientOptFn) *REDClient {
+	opt := metricOpts{
+		namespace: "service",
+		service:   service,
+		counterMetrics: map[string]VecOpts{
+			"call_total": {
+				Help:       "Number of calls",
+				LabelNames: []string{"method"},
+				CounterFn: func(vec *prometheus.CounterVec, o CollectFnOpts) {
+					vec.With(prometheus.Labels{"method": o.Method}).Inc()
+				},
+			},
+			"error_total": {
+				Help:       "Number of errors encountered",
+				LabelNames: []string{"method", "code"},
+				CounterFn: func(vec *prometheus.CounterVec, o CollectFnOpts) {
+					if o.Err != nil {
+						vec.With(prometheus.Labels{
+							"method": o.Method,
+							"code":   influxdb.ErrorCode(o.Err),
+						}).Inc()
+					}
+				},
+			},
+		},
+		histogramMetrics: map[string]VecOpts{
+			"duration": {
+				Help:       "Duration of calls",
+				LabelNames: []string{"method"},
+				HistogramFn: func(vec *prometheus.HistogramVec, o CollectFnOpts) {
+					vec.
+						With(prometheus.Labels{"method": o.Method}).
+						Observe(time.Since(o.Start).Seconds())
+				},
+			},
+		},
+	}
+	for _, o := range opts {
+		o(&opt)
+	}
+
+	client := new(REDClient)
+	for metricName, vecOpts := range opt.counterMetrics {
+		client.metrics = append(client.metrics, &counter{
+			fn: vecOpts.CounterFn,
+			CounterVec: prometheus.NewCounterVec(prometheus.CounterOpts{
+				Namespace: opt.namespace,
+				Subsystem: opt.serviceName(),
+				Name:      metricName,
+				Help:      vecOpts.Help,
+			}, vecOpts.LabelNames),
+		})
+	}
+
+	for metricName, vecOpts := range opt.histogramMetrics {
+		client.metrics = append(client.metrics, &histogram{
+			fn: vecOpts.HistogramFn,
+			HistogramVec: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+				Namespace: opt.namespace,
+				Subsystem: opt.serviceName(),
+				Name:      metricName,
+				Help:      vecOpts.Help,
+			}, vecOpts.LabelNames),
+		})
+	}
+
+	reg.MustRegister(client.collectors()...)
+
+	return client
+}
+
+type RecordFn func(err error, opts ...func(opts *CollectFnOpts)) error
+
+// RecordAdditional provides an extension to the base method, err data provided
+// to the metrics.
+func RecordAdditional(props map[string]interface{}) func(opts *CollectFnOpts) {
+	return func(opts *CollectFnOpts) {
+		opts.AdditionalProps = props
 	}
 }
 
 // Record returns a record fn that is called on any given return err. If an error is encountered
 // it will register the err metric. The err is never altered.
-func (c *REDClient) Record(method string) func(error) error {
+func (c *REDClient) Record(method string) RecordFn {
 	start := time.Now()
-	return func(err error) error {
-		c.reqs.With(prometheus.Labels{"method": method}).Inc()
-
-		if err != nil {
-			c.errs.With(prometheus.Labels{
-				"method": method,
-				"code":   influxdb.ErrorCode(err),
-			}).Inc()
+	return func(err error, opts ...func(opts *CollectFnOpts)) error {
+		opt := CollectFnOpts{
+			Method: method,
+			Start:  start,
+			Err:    err,
+		}
+		for _, o := range opts {
+			o(&opt)
 		}
 
-		c.durs.With(prometheus.Labels{"method": method}).Observe(time.Since(start).Seconds())
+		for _, metric := range c.metrics {
+			metric.collect(opt)
+		}
 
 		return err
 	}
 }
+
+func (c *REDClient) collectors() []prometheus.Collector {
+	var collectors []prometheus.Collector
+	for _, metric := range c.metrics {
+		collectors = append(collectors, metric)
+	}
+	return collectors
+}
+
+type metricCollector interface {
+	prometheus.Collector
+
+	collect(o CollectFnOpts)
+}
+
+type counter struct {
+	*prometheus.CounterVec
+
+	fn CounterFn
+}
+
+func (c *counter) collect(o CollectFnOpts) {
+	c.fn(c.CounterVec, o)
+}
+
+type histogram struct {
+	*prometheus.HistogramVec
+
+	fn HistogramFn
+}
+
+func (h *histogram) collect(o CollectFnOpts) {
+	h.fn(h.HistogramVec, o)
+}

kit/metric/metrics_options.go

@@ -1,36 +1,84 @@
 package metric
 
-import "fmt"
+import (
+	"fmt"
+	"time"
 
-type metricOpts struct {
-	serviceSuffix string
-}
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type (
+	// CollectFnOpts provides arugments to the collect operation of a metric.
+	CollectFnOpts struct {
+		Method          string
+		Start           time.Time
+		Err             error
+		AdditionalProps map[string]interface{}
+	}
+
+	CounterFn func(vec *prometheus.CounterVec, o CollectFnOpts)
+
+	HistogramFn func(vec *prometheus.HistogramVec, o CollectFnOpts)
 
-func defaultOpts() *metricOpts {
-	return &metricOpts{}
+	// VecOpts expands on the
+	VecOpts struct {
+		Name       string
+		Help       string
+		LabelNames []string
+
+		CounterFn   CounterFn
+		HistogramFn HistogramFn
+	}
+)
+
+type metricOpts struct {
+	namespace        string
+	service          string
+	serviceSuffix    string
+	counterMetrics   map[string]VecOpts
+	histogramMetrics map[string]VecOpts
 }
 
-func (o *metricOpts) ApplySuffix(prefix string) string {
+func (o metricOpts) serviceName() string {
 	if o.serviceSuffix != "" {
-		return fmt.Sprintf("%s_%s", prefix, o.serviceSuffix)
+		return fmt.Sprintf("%s_%s", o.service, o.serviceSuffix)
 	}
-	return prefix
+	return o.service
 }
 
-// MetricsOption is an option used by a metric middleware.
-type MetricsOption func(*metricOpts)
+// ClientOptFn is an option used by a metric middleware.
+type ClientOptFn func(*metricOpts)
+
+// WithVec sets a new counter vector to be collected.
+func WithVec(opts VecOpts) ClientOptFn {
+	return func(o *metricOpts) {
+		if opts.CounterFn != nil {
+			if o.counterMetrics == nil {
+				o.counterMetrics = make(map[string]VecOpts)
+			}
+			o.counterMetrics[opts.Name] = opts
+		}
+	}
+}
 
 // WithSuffix returns a metric option that applies a suffix to the service name of the metric.
-func WithSuffix(suffix string) MetricsOption {
+func WithSuffix(suffix string) ClientOptFn {
 	return func(opts *metricOpts) {
 		opts.serviceSuffix = suffix
 	}
 }
 
-func ApplyMetricOpts(opts ...MetricsOption) *metricOpts {
-	o := defaultOpts()
+func ApplyMetricOpts(opts ...ClientOptFn) *metricOpts {
+	o := metricOpts{}
 	for _, opt := range opts {
-		opt(o)
+		opt(&o)
 	}
-	return o
+	return &o
+}
+
+func (o *metricOpts) ApplySuffix(prefix string) string {
+	if o.serviceSuffix != "" {
+		return fmt.Sprintf("%s_%s", prefix, o.serviceSuffix)
+	}
+	return prefix
 }

label/middleware_metrics.go

@@ -15,7 +15,7 @@ type LabelMetrics struct {
 	labelService influxdb.LabelService
 }
 
-func NewLabelMetrics(reg prometheus.Registerer, s influxdb.LabelService, opts ...metric.MetricsOption) *LabelMetrics {
+func NewLabelMetrics(reg prometheus.Registerer, s influxdb.LabelService, opts ...metric.ClientOptFn) *LabelMetrics {
 	o := metric.ApplyMetricOpts(opts...)
 	return &LabelMetrics{
 		rec:          metric.New(reg, o.ApplySuffix("org")),