Remove metric recreation when filtering (influxdata#4767)

agent/accumulator.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/metric"
 	"github.com/influxdata/telegraf/selfstat"
 )
 
@@ -14,13 +15,13 @@ var (
 
 type MetricMaker interface {
 	Name() string
-	MakeMetric(
-		measurement string,
-		fields map[string]interface{},
-		tags map[string]string,
-		mType telegraf.ValueType,
-		t time.Time,
-	) telegraf.Metric
+	MakeMetric(metric telegraf.Metric) telegraf.Metric
+}
+
+type accumulator struct {
+	maker     MetricMaker
+	metrics   chan telegraf.Metric
+	precision time.Duration
 }
 
 func NewAccumulator(
@@ -35,23 +36,13 @@ func NewAccumulator(
 	return &acc
 }
 
-type accumulator struct {
-	metrics chan telegraf.Metric
-
-	maker MetricMaker
-
-	precision time.Duration
-}
-
 func (ac *accumulator) AddFields(
 	measurement string,
 	fields map[string]interface{},
 	tags map[string]string,
 	t ...time.Time,
 ) {
-	if m := ac.maker.MakeMetric(measurement, fields, tags, telegraf.Untyped, ac.getTime(t)); m != nil {
-		ac.metrics <- m
-	}
+	ac.addMetric(measurement, tags, fields, telegraf.Untyped, t...)
 }
 
 func (ac *accumulator) AddGauge(
@@ -60,9 +51,7 @@ func (ac *accumulator) AddGauge(
 	tags map[string]string,
 	t ...time.Time,
 ) {
-	if m := ac.maker.MakeMetric(measurement, fields, tags, telegraf.Gauge, ac.getTime(t)); m != nil {
-		ac.metrics <- m
-	}
+	ac.addMetric(measurement, tags, fields, telegraf.Gauge, t...)
 }
 
 func (ac *accumulator) AddCounter(
@@ -71,9 +60,7 @@ func (ac *accumulator) AddCounter(
 	tags map[string]string,
 	t ...time.Time,
 ) {
-	if m := ac.maker.MakeMetric(measurement, fields, tags, telegraf.Counter, ac.getTime(t)); m != nil {
-		ac.metrics <- m
-	}
+	ac.addMetric(measurement, tags, fields, telegraf.Counter, t...)
 }
 
 func (ac *accumulator) AddSummary(
@@ -82,9 +69,7 @@ func (ac *accumulator) AddSummary(
 	tags map[string]string,
 	t ...time.Time,
 ) {
-	if m := ac.maker.MakeMetric(measurement, fields, tags, telegraf.Summary, ac.getTime(t)); m != nil {
-		ac.metrics <- m
-	}
+	ac.addMetric(measurement, tags, fields, telegraf.Summary, t...)
 }
 
 func (ac *accumulator) AddHistogram(
@@ -93,7 +78,21 @@ func (ac *accumulator) AddHistogram(
 	tags map[string]string,
 	t ...time.Time,
 ) {
-	if m := ac.maker.MakeMetric(measurement, fields, tags, telegraf.Histogram, ac.getTime(t)); m != nil {
+	ac.addMetric(measurement, tags, fields, telegraf.Histogram, t...)
+}
+
+func (ac *accumulator) addMetric(
+	measurement string,
+	tags map[string]string,
+	fields map[string]interface{},
+	tp telegraf.ValueType,
+	t ...time.Time,
+) {
+	m, err := metric.New(measurement, tags, fields, ac.getTime(t), tp)
+	if err != nil {
+		return
+	}
+	if m := ac.maker.MakeMetric(m); m != nil {
 		ac.metrics <- m
 	}
 }
@@ -105,7 +104,6 @@ func (ac *accumulator) AddError(err error) {
 		return
 	}
 	NErrors.Incr(1)
-	//TODO suppress/throttle consecutive duplicate errors?
 	log.Printf("E! Error in plugin [%s]: %s", ac.maker.Name(), err)
 }
 

agent/accumulator_test.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/metric"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -134,26 +133,7 @@ type TestMetricMaker struct {
 func (tm *TestMetricMaker) Name() string {
 	return "TestPlugin"
 }
-func (tm *TestMetricMaker) MakeMetric(
-	measurement string,
-	fields map[string]interface{},
-	tags map[string]string,
-	mType telegraf.ValueType,
-	t time.Time,
-) telegraf.Metric {
-	switch mType {
-	case telegraf.Untyped:
-		if m, err := metric.New(measurement, tags, fields, t); err == nil {
-			return m
-		}
-	case telegraf.Counter:
-		if m, err := metric.New(measurement, tags, fields, t, telegraf.Counter); err == nil {
-			return m
-		}
-	case telegraf.Gauge:
-		if m, err := metric.New(measurement, tags, fields, t, telegraf.Gauge); err == nil {
-			return m
-		}
-	}
-	return nil
+
+func (tm *TestMetricMaker) MakeMetric(metric telegraf.Metric) telegraf.Metric {
+	return metric
 }

docs/CONFIGURATION.md

@@ -1,11 +1,14 @@
-# Telegraf Configuration
+# Configuration
 
-You can see the latest config file with all available plugins here:
-[telegraf.conf](https://github.com/influxdata/telegraf/blob/master/etc/telegraf.conf)
+Telegraf's configuration file is written using
+[TOML](https://github.com/toml-lang/toml#toml).
+
+[View the telegraf.conf config file with all available
+plugins](/etc/telegraf.conf).
 
 ## Generating a Configuration File
 
-A default Telegraf config file can be auto-generated by telegraf:
+A default config file can be generated by telegraf:
 
 ```
 telegraf config > telegraf.conf
@@ -18,7 +21,7 @@ To generate a file with specific inputs and outputs, you can use the
 telegraf --input-filter cpu:mem:net:swap --output-filter influxdb:kafka config
 ```
 
-## Environment Variables
+### Environment Variables
 
 Environment variables can be used anywhere in the config file, simply prepend
 them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
@@ -27,7 +30,7 @@ for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)
 When using the `.deb` or `.rpm` packages, you can define environment variables
 in the `/etc/default/telegraf` file.
 
-## Configuration file locations
+### Configuration file locations
 
 The location of the configuration file can be set via the `--config` command
 line flag.
@@ -40,13 +43,13 @@ On most systems, the default locations are `/etc/telegraf/telegraf.conf` for
 the main configuration file and `/etc/telegraf/telegraf.d` for the directory of
 configuration files.
 
-# Global Tags
+### Global Tags
 
 Global tags can be specified in the `[global_tags]` section of the config file
 in key="value" format. All metrics being gathered on this host will be tagged
 with the tags specified here.
 
-## Agent Configuration
+### Agent Configuration
 
 Telegraf has a few options you can configure under the `[agent]` section of the
 config.
@@ -85,7 +88,7 @@ ie, a jitter of 5s and flush_interval 10s means flushes will happen every 10-15s
 * **hostname**: Override default hostname, if empty use os.Hostname().
 * **omit_hostname**: If true, do no set the "host" tag in the telegraf agent.
 
-## Input Configuration
+### Input Configuration
 
 The following config parameters are available for all inputs:
 
@@ -98,15 +101,15 @@ you can configure that here.
 * **name_suffix**: Specifies a suffix to attach to the measurement name.
 * **tags**: A map of tags to apply to a specific input's measurements.
 
-The [measurement filtering](#measurement-filtering) parameters can be used to
-limit what metrics are emitted from the input plugin.
+The [metric filtering](#metric-filtering) parameters can be used to limit what metrics are
+emitted from the input plugin.
 
-## Output Configuration
+### Output Configuration
 
-The [measurement filtering](#measurement-filtering) parameters can be used to
-limit what metrics are emitted from the output plugin.
+The [metric filtering](#metric-filtering) parameters can be used to limit what metrics are
+emitted from the output plugin.
 
-## Aggregator Configuration
+### Aggregator Configuration
 
 The following config parameters are available for all aggregators:
 
@@ -125,63 +128,77 @@ aggregator and will not get sent to the output plugins.
 * **name_suffix**: Specifies a suffix to attach to the measurement name.
 * **tags**: A map of tags to apply to a specific input's measurements.
 
-The [measurement filtering](#measurement-filtering) parameters can be used to
-limit what metrics are handled by the aggregator.  Excluded metrics are passed
-downstream to the next aggregator.
+The [metric filtering](#metric-filtering) parameters can be used to limit what metrics are
+handled by the aggregator.  Excluded metrics are passed downstream to the next
+aggregator.
 
-## Processor Configuration
+### Processor Configuration
 
 The following config parameters are available for all processors:
 
 * **order**: This is the order in which the processor(s) get executed. If this
 is not specified then processor execution order will be random.
 
-The [measurement filtering](#measurement-filtering) parameters can be used
-to limit what metrics are handled by the processor.  Excluded metrics are
-passed downstream to the next processor.
+The [metric filtering](#metric-filtering) parameters can be used to limit what metrics are
+handled by the processor.  Excluded metrics are passed downstream to the next
+processor.
+
+<a id="measurement-filtering"></a>
+### Metric Filtering
 
-#### Measurement Filtering
+Metric filtering can be configured per plugin on any input, output, processor,
+and aggregator plugin.  Filters fall under two categories: Selectors and
+Modifiers.
 
-Filters can be configured per input, output, processor, or aggregator,
-see below for examples.
+#### Selectors
 
-* **namepass**:
-An array of glob pattern strings.  Only points whose measurement name matches
+Selector filters include or exclude entire metrics.  When a metric is excluded
+from a Input or an Output plugin, the metric is dropped.  If a metric is
+excluded from a Processor or Aggregator plugin, it is skips the plugin and is
+sent onwards to the next stage of processing.
+
+- **namepass**:
+An array of glob pattern strings.  Only metrics whose measurement name matches
 a pattern in this list are emitted.
-* **namedrop**:
-The inverse of `namepass`.  If a match is found the point is discarded. This
-is tested on points after they have passed the `namepass` test.
-* **fieldpass**:
+
+- **namedrop**:
+The inverse of `namepass`.  If a match is found the metric is discarded. This
+is tested on metrics after they have passed the `namepass` test.
+
+- **tagpass**:
+A table mapping tag keys to arrays of glob pattern strings.  Only metrics
+that contain a tag key in the table and a tag value matching one of its
+patterns is emitted.
+
+- **tagdrop**:
+The inverse of `tagpass`.  If a match is found the metric is discarded. This
+is tested on metrics after they have passed the `tagpass` test.
+
+#### Modifiers
+
+Modifier filters remove tags and fields from a metric.  If all fields are
+removed the metric is removed.
+
+- **fieldpass**:
 An array of glob pattern strings.  Only fields whose field key matches a
 pattern in this list are emitted.
-* **fielddrop**:
+
+- **fielddrop**:
 The inverse of `fieldpass`.  Fields with a field key matching one of the
-patterns will be discarded from the point.  This is tested on points after
+patterns will be discarded from the metric.  This is tested on metrics after
 they have passed the `fieldpass` test.
-* **tagpass**:
-A table mapping tag keys to arrays of glob pattern strings.  Only points
-that contain a tag key in the table and a tag value matching one of its
-patterns is emitted.
-* **tagdrop**:
-The inverse of `tagpass`.  If a match is found the point is discarded. This
-is tested on points after they have passed the `tagpass` test.
-* **taginclude**:
+
+- **taginclude**:
 An array of glob pattern strings.  Only tags with a tag key matching one of
 the patterns are emitted.  In contrast to `tagpass`, which will pass an entire
-point based on its tag, `taginclude` removes all non matching tags from the
-point.  This filter can be used on both inputs & outputs, but it is
-_recommended_ to be used on inputs, as it is more efficient to filter out tags
-at the ingestion point.
-* **tagexclude**:
-The inverse of `taginclude`. Tags with a tag key matching one of the patterns
-will be discarded from the point.
+metric based on its tag, `taginclude` removes all non matching tags from the
+metric.
 
-**NOTE** Due to the way TOML is parsed, `tagpass` and `tagdrop` parameters
-must be defined at the _end_ of the plugin definition, otherwise subsequent
-plugin config options will be interpreted as part of the tagpass/tagdrop
-tables.
+- **tagexclude**:
+The inverse of `taginclude`. Tags with a tag key matching one of the patterns
+will be discarded from the metric.
 
-#### Input Configuration Examples
+### Input Configuration Examples
 
 This is a full working config that will output CPU data to an InfluxDB instance
 at 192.168.59.103:8086, tagging measurements with dc="denver-1". It will output

internal/config/config.go

@@ -884,14 +884,6 @@ func (c *Config) addInput(name string, table *ast.Table) error {
 // builds the filter and returns a
 // models.AggregatorConfig to be inserted into models.RunningAggregator
 func buildAggregator(name string, tbl *ast.Table) (*models.AggregatorConfig, error) {
-	unsupportedFields := []string{"tagexclude", "taginclude"}
-	for _, field := range unsupportedFields {
-		if _, ok := tbl.Fields[field]; ok {
-			return nil, fmt.Errorf("%s is not supported for aggregator plugins (%s).",
-				field, name)
-		}
-	}
-
 	conf := &models.AggregatorConfig{
 		Name:   name,
 		Delay:  time.Millisecond * 100,
@@ -989,13 +981,6 @@ func buildAggregator(name string, tbl *ast.Table) (*models.AggregatorConfig, err
 // models.ProcessorConfig to be inserted into models.RunningProcessor
 func buildProcessor(name string, tbl *ast.Table) (*models.ProcessorConfig, error) {
 	conf := &models.ProcessorConfig{Name: name}
-	unsupportedFields := []string{"tagexclude", "taginclude", "fielddrop", "fieldpass"}
-	for _, field := range unsupportedFields {
-		if _, ok := tbl.Fields[field]; ok {
-			return nil, fmt.Errorf("%s is not supported for processor plugins (%s).",
-				field, name)
-		}
-	}
 
 	if node, ok := tbl.Fields["order"]; ok {
 		if kv, ok := node.(*ast.KeyValue); ok {