Merge pull request #1300 from GeorgeTsagk/prometheus-bump-timeout

monitoring: export collector timeout as prometheus config

Author: guggero

monitoring/asset_balances_collector.go

@@ -72,7 +72,7 @@ func (a *assetBalancesCollector) Collect(ch chan<- prometheus.Metric) {
 	a.collectMx.Lock()
 	defer a.collectMx.Unlock()
 
-	ctxdb, cancel := context.WithTimeout(context.Background(), dbTimeout)
+	ctxdb, cancel := context.WithTimeout(context.Background(), promTimeout)
 	defer cancel()
 
 	assets, err := a.cfg.AssetStore.FetchAllAssets(ctxdb, false, false, nil)

monitoring/asset_collector.go

@@ -97,7 +97,7 @@ func (a *universeStatsCollector) Collect(ch chan<- prometheus.Metric) {
 	a.collectMx.Lock()
 	defer a.collectMx.Unlock()
 
-	ctx, cancel := context.WithTimeout(context.Background(), dbTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), promTimeout)
 	defer cancel()
 
 	universeStats, err := a.cfg.UniverseStats.AggregateSyncStats(ctx)

monitoring/config.go

@@ -1,6 +1,8 @@
 package monitoring
 
 import (
+	"time"
+
 	"github.com/lightninglabs/taproot-assets/tapdb"
 	"github.com/lightninglabs/taproot-assets/tapgarden"
 	"github.com/lightninglabs/taproot-assets/universe"
@@ -10,6 +12,8 @@ import (
 // PrometheusConfig is the set of configuration data that specifies if
 // Prometheus metric exporting is activated, and if so the listening address of
 // the Prometheus server.
+//
+// nolint: lll
 type PrometheusConfig struct {
 	// Active, if true, then Prometheus metrics will be exported.
 	Active bool `long:"active" description:"if true prometheus metrics will be exported"`
@@ -18,6 +22,11 @@ type PrometheusConfig struct {
 	// main Prometheus server to scrape our metrics.
 	ListenAddr string `long:"listenaddr" description:"the interface we should listen on for prometheus"`
 
+	// CollectorRPCTimeout is the context timeout to be used by the RPC
+	// calls performed during metrics collection. This should not be greater
+	// than the scrape interval of prometheus.
+	CollectorRPCTimeout time.Duration `long:"collector-rpc-timeout" description:"the default timeout to be used in the RPC calls performed during metric collection"`
+
 	// RPCServer is a pointer to the main RPC server. We use this to export
 	// generic RPC metrics to monitor the health of the service.
 	RPCServer *grpc.Server
@@ -45,7 +54,8 @@ type PrometheusConfig struct {
 // metrics exporter.
 func DefaultPrometheusConfig() PrometheusConfig {
 	return PrometheusConfig{
-		ListenAddr: "127.0.0.1:8989",
-		Active:     false,
+		ListenAddr:          "127.0.0.1:8989",
+		Active:              false,
+		CollectorRPCTimeout: defaultTimeout,
 	}
 }

monitoring/db_collector.go

@@ -84,7 +84,7 @@ func (a *dbCollector) Collect(ch chan<- prometheus.Metric) {
 	a.collectMx.Lock()
 	defer a.collectMx.Unlock()
 
-	ctxdb, cancel := context.WithTimeout(context.Background(), dbTimeout)
+	ctxdb, cancel := context.WithTimeout(context.Background(), promTimeout)
 	defer cancel()
 
 	// Fetch the db size.

monitoring/prometheus.go

@@ -12,15 +12,18 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
+const (
+	// defaultTimeout is the default timeout.
+	defaultTimeout = 25 * time.Second
+)
+
 var (
 	// serverMetrics is a global variable that holds the Prometheus metrics
 	// for the gRPC server.
 	serverMetrics *grpc_prometheus.ServerMetrics
-)
 
-const (
-	// dbTimeout is the default database timeout.
-	dbTimeout = 20 * time.Second
+	// promTimeout is the timeout used by the prometheus collectors.
+	promTimeout time.Duration
 )
 
 // PrometheusExporter is a metric exporter that uses Prometheus directly. The
@@ -46,6 +49,11 @@ func (p *PrometheusExporter) Start() error {
 		return fmt.Errorf("server metrics not set")
 	}
 
+	promTimeout = defaultTimeout
+	if p.config.CollectorRPCTimeout.Seconds() != 0 {
+		promTimeout = p.config.CollectorRPCTimeout
+	}
+
 	// Create a custom Prometheus registry.
 	p.registry = prometheus.NewRegistry()
 	p.registry.MustRegister(collectors.NewProcessCollector(

sample-tapd.conf

@@ -346,6 +346,9 @@
 ; The interface we should listen on for prometheus
 ; prometheus.listenaddr=127.0.0.1:8989
 
+; The default timeout used in prometheus collectors.
+; prometheus.collector-rpc-timeout=25s
+
 ; Enable additional histogram to track gRPC call processing performance
 ; (latency, etc)
 ; prometheus.perfhistograms=false