PMM-4145 Add RDS disable metric collection.

README.md

@@ -27,6 +27,8 @@ instances:
     instance: rds-mysql57
     aws_access_key: AKIAIOSFODNN7EXAMPLE
     aws_secret_key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+    disable_basic_metrics: true
+    disable_enhanced_metrics: false
     labels:
       foo: bar
       baz: qux

basic/collector.go

@@ -61,9 +61,13 @@ func (e *Collector) collect(ch chan<- prometheus.Metric) {
 	var wg sync.WaitGroup
 	defer wg.Wait()
 
-	wg.Add(len(e.config.Instances))
 	for _, instance := range e.config.Instances {
+		if instance.DisableBasicMetrics {
+			e.l.Debugf("Instance %s has disabled basic metrics, skipping.", instance)
+			continue
+		}
 		instance := instance
+		wg.Add(1)
 		go func() {
 			defer wg.Done()
 

basic/collector_test.go

@@ -1,7 +1,9 @@
 package basic
 
 import (
+	"fmt"
 	"sort"
+	"strings"
 	"testing"
 
 	"github.com/percona/exporter_shared/helpers"
@@ -46,3 +48,45 @@ func TestCollector(t *testing.T) {
 	assert.Equal(t, expectedLines, actualLines)
 	assert.Equal(t, expectedMetrics, actualMetrics)
 }
+
+func TestCollectorDisableBasicMetrics(t *testing.T) {
+	cfg, err := config.Load("../config.tests.yml")
+	require.NoError(t, err)
+	client := client.New()
+	instanceGroups := make(map[bool][]string, 2)
+	for i := range cfg.Instances {
+		// Disable basic metrics in even instances.
+		// This disable instance: no-such-instance.
+		isDisabled := i%2 == 0
+		cfg.Instances[i].DisableBasicMetrics = isDisabled
+		// Groups instance names by disabled or enabled metrics.
+		instanceGroups[isDisabled] = append(instanceGroups[isDisabled], cfg.Instances[i].Instance)
+	}
+	sess, err := sessions.New(cfg.Instances, client.HTTP(), false)
+	require.NoError(t, err)
+
+	c := New(cfg, sess)
+
+	actualMetrics := helpers.ReadMetrics(helpers.CollectMetrics(c))
+	actualLines := helpers.Format(helpers.WriteMetrics(actualMetrics))
+
+	// Check if all collected metrics do not contain metrics for instance whare disabled metrics.
+	hasMetricForInstance := func(lines []string, instanceName string) bool {
+		for _, line := range lines {
+			if strings.Contains(line, fmt.Sprintf("instance=%q", instanceName)) {
+				return true
+			}
+		}
+		return false
+	}
+
+	// Scans if metrics contain a metric for the disabled instance (DisableBasicMetrics = true).
+	for _, inst := range instanceGroups[true] {
+		assert.Falsef(t, hasMetricForInstance(actualLines, inst), "Found metrics for disabled instance %s", inst)
+	}
+
+	// Scans if metrics contain a metric for the enabled instance (DisableBasicMetrics = false).
+	for _, inst := range instanceGroups[false] {
+		assert.Truef(t, hasMetricForInstance(actualLines, inst), "Did not find metrics for enabled instance %s", inst)
+	}
+}

config/config.go

@@ -18,11 +18,13 @@ import (
 
 // Instance represents a single RDS information from configuration file.
 type Instance struct {
-	Region       string            `yaml:"region"`
-	Instance     string            `yaml:"instance"`
-	AWSAccessKey string            `yaml:"aws_access_key"` // may be empty
-	AWSSecretKey string            `yaml:"aws_secret_key"` // may be empty
-	Labels       map[string]string `yaml:"labels"`         // may be empty
+	Region                 string            `yaml:"region"`
+	Instance               string            `yaml:"instance"`
+	AWSAccessKey           string            `yaml:"aws_access_key"` // may be empty
+	AWSSecretKey           string            `yaml:"aws_secret_key"` // may be empty
+	DisableBasicMetrics    bool              `yaml:"disable_basic_metrics"`
+	DisableEnhancedMetrics bool              `yaml:"disable_enhanced_metrics"`
+	Labels                 map[string]string `yaml:"labels"` // may be empty
 
 	// TODO Type InstanceType `yaml:"type"` // may be empty for old pmm-managed
 }

enhanced/scraper.go

@@ -89,6 +89,11 @@ func (s *scraper) scrape(ctx context.Context) (map[string][]prometheus.Metric, m
 				l.Errorf("Failed to find instance.")
 				continue
 			}
+
+			if instance.DisableEnhancedMetrics {
+				l.Debugf("Enhanced Metrics are dissabled for instance %v.", instance)
+				continue
+			}
 			l = l.With("region", instance.Region).With("instance", instance.Instance)
 
 			// l.Debugf("Message:\n%s", *event.Message)

enhanced/scraper_test.go

@@ -135,3 +135,47 @@ func TestBetterTimes(t *testing.T) {
 		assert.Equal(t, td.expectedNextStartTime, nextStartTime)
 	}
 }
+
+func TestScraperDisableEnhancedMetrics(t *testing.T) {
+	cfg, err := config.Load("../config.tests.yml")
+	require.NoError(t, err)
+	client := client.New()
+	for i := range cfg.Instances {
+		// Disable enhanced metrics in even instances.
+		// This disable instance: no-such-instance.
+		isDisabled := i%2 == 0
+		cfg.Instances[i].DisableEnhancedMetrics = isDisabled
+	}
+	sess, err := sessions.New(cfg.Instances, client.HTTP(), false)
+	require.NoError(t, err)
+
+	// Check if all collected metrics do not contain metrics for instance with disabled metrics.
+	hasMetricForInstance := func(lines []string, instanceName string) bool {
+		for _, line := range lines {
+			if strings.Contains(line, fmt.Sprintf("instance=%q", instanceName)) {
+				return true
+			}
+		}
+		return false
+	}
+
+	for session, instances := range sess.AllSessions() {
+		session, instances := session, instances
+		t.Run(fmt.Sprint(instances), func(t *testing.T) {
+			s := newScraper(session, instances)
+			s.testDisallowUnknownFields = true
+			metrics, _ := s.scrape(context.Background())
+
+			for _, instance := range instances {
+				actualMetrics := helpers.ReadMetrics(metrics[instance.ResourceID])
+				actualLines := helpers.Format(helpers.WriteMetrics(actualMetrics))
+				name := instance.Instance
+				if instance.DisableEnhancedMetrics {
+					assert.Falsef(t, hasMetricForInstance(actualLines, name), "Found metrics for disabled instance %s", name)
+					continue
+				}
+				assert.Truef(t, hasMetricForInstance(actualLines, name), "Did not find metrics for enabled instance %s", name)
+			}
+		})
+	}
+}

sessions/sessions.go

@@ -20,6 +20,8 @@ import (
 type Instance struct {
 	Region                     string
 	Instance                   string
+	DisableBasicMetrics        bool
+	DisableEnhancedMetrics     bool
 	ResourceID                 string
 	Labels                     map[string]string
 	EnhancedMonitoringInterval time.Duration
@@ -52,9 +54,11 @@ func New(instances []config.Instance, client *http.Client, trace bool) (*Session
 		// re-use session for the same region and key (explicit or empty for implicit) pair
 		if s := sharedSessions[instance.Region+"/"+instance.AWSAccessKey]; s != nil {
 			res.sessions[s] = append(res.sessions[s], Instance{
-				Region:   instance.Region,
-				Instance: instance.Instance,
-				Labels:   instance.Labels,
+				Region:                 instance.Region,
+				Instance:               instance.Instance,
+				Labels:                 instance.Labels,
+				DisableBasicMetrics:    instance.DisableBasicMetrics,
+				DisableEnhancedMetrics: instance.DisableEnhancedMetrics,
 			})
 			continue
 		}
@@ -96,9 +100,11 @@ func New(instances []config.Instance, client *http.Client, trace bool) (*Session
 		}
 		sharedSessions[instance.Region+"/"+instance.AWSAccessKey] = s
 		res.sessions[s] = append(res.sessions[s], Instance{
-			Region:   instance.Region,
-			Instance: instance.Instance,
-			Labels:   instance.Labels,
+			Region:                 instance.Region,
+			Instance:               instance.Instance,
+			Labels:                 instance.Labels,
+			DisableBasicMetrics:    instance.DisableBasicMetrics,
+			DisableEnhancedMetrics: instance.DisableEnhancedMetrics,
 		})
 	}