feat(config): Deprecate fieldpass and fielddrop modifiers (influx…

…data#14012)

config/config.go

@@ -1289,7 +1289,7 @@ func (c *Config) buildAggregator(name string, tbl *ast.Table) (*models.Aggregato
 	}
 
 	var err error
-	conf.Filter, err = c.buildFilter(tbl)
+	conf.Filter, err = c.buildFilter("aggregators."+name, tbl)
 	if err != nil {
 		return conf, err
 	}
@@ -1313,7 +1313,7 @@ func (c *Config) buildProcessor(category, name string, tbl *ast.Table) (*models.
 	}
 
 	var err error
-	conf.Filter, err = c.buildFilter(tbl)
+	conf.Filter, err = c.buildFilter(category+"."+name, tbl)
 	if err != nil {
 		return conf, err
 	}
@@ -1324,22 +1324,60 @@ func (c *Config) buildProcessor(category, name string, tbl *ast.Table) (*models.
 }
 
 // buildFilter builds a Filter
-// (tagpass/tagdrop/namepass/namedrop/fieldpass/fielddrop) to
+// (tags, fields, namepass, namedrop, metricpass) to
 // be inserted into the models.OutputConfig/models.InputConfig
 // to be used for glob filtering on tags and measurements
-func (c *Config) buildFilter(tbl *ast.Table) (models.Filter, error) {
+func (c *Config) buildFilter(plugin string, tbl *ast.Table) (models.Filter, error) {
 	f := models.Filter{}
 
 	c.getFieldStringSlice(tbl, "namepass", &f.NamePass)
 	c.getFieldString(tbl, "namepass_separator", &f.NamePassSeparators)
 	c.getFieldStringSlice(tbl, "namedrop", &f.NameDrop)
 	c.getFieldString(tbl, "namedrop_separator", &f.NameDropSeparators)
 
-	c.getFieldStringSlice(tbl, "pass", &f.FieldPass)
-	c.getFieldStringSlice(tbl, "fieldpass", &f.FieldPass)
+	var oldPass []string
+	c.getFieldStringSlice(tbl, "pass", &oldPass)
+	if len(oldPass) > 0 {
+		models.PrintOptionDeprecationNotice(telegraf.Warn, plugin, "pass", telegraf.DeprecationInfo{
+			Since:     "0.10.4",
+			RemovalIn: "2.0.0",
+			Notice:    "use 'fieldinclude' instead",
+		})
+		f.FieldInclude = append(f.FieldInclude, oldPass...)
+	}
+	var oldFieldPass []string
+	c.getFieldStringSlice(tbl, "fieldpass", &oldFieldPass)
+	if len(oldFieldPass) > 0 {
+		models.PrintOptionDeprecationNotice(telegraf.Warn, plugin, "fieldpass", telegraf.DeprecationInfo{
+			Since:     "1.29.0",
+			RemovalIn: "2.0.0",
+			Notice:    "use 'fieldinclude' instead",
+		})
+		f.FieldInclude = append(f.FieldInclude, oldFieldPass...)
+	}
+	c.getFieldStringSlice(tbl, "fieldinclude", &f.FieldInclude)
 
-	c.getFieldStringSlice(tbl, "drop", &f.FieldDrop)
-	c.getFieldStringSlice(tbl, "fielddrop", &f.FieldDrop)
+	var oldDrop []string
+	c.getFieldStringSlice(tbl, "drop", &oldDrop)
+	if len(oldDrop) > 0 {
+		models.PrintOptionDeprecationNotice(telegraf.Warn, plugin, "drop", telegraf.DeprecationInfo{
+			Since:     "0.10.4",
+			RemovalIn: "2.0.0",
+			Notice:    "use 'fieldexclude' instead",
+		})
+		f.FieldExclude = append(f.FieldExclude, oldDrop...)
+	}
+	var oldFieldDrop []string
+	c.getFieldStringSlice(tbl, "fielddrop", &oldFieldDrop)
+	if len(oldFieldDrop) > 0 {
+		models.PrintOptionDeprecationNotice(telegraf.Warn, plugin, "fielddrop", telegraf.DeprecationInfo{
+			Since:     "1.29.0",
+			RemovalIn: "2.0.0",
+			Notice:    "use 'fieldexclude' instead",
+		})
+		f.FieldExclude = append(f.FieldExclude, oldFieldDrop...)
+	}
+	c.getFieldStringSlice(tbl, "fieldexclude", &f.FieldExclude)
 
 	c.getFieldTagFilter(tbl, "tagpass", &f.TagPassFilters)
 	c.getFieldTagFilter(tbl, "tagdrop", &f.TagDropFilters)
@@ -1392,7 +1430,7 @@ func (c *Config) buildInput(name string, tbl *ast.Table) (*models.InputConfig, e
 	}
 
 	var err error
-	cp.Filter, err = c.buildFilter(tbl)
+	cp.Filter, err = c.buildFilter("inputs."+name, tbl)
 	if err != nil {
 		return cp, err
 	}
@@ -1407,7 +1445,7 @@ func (c *Config) buildInput(name string, tbl *ast.Table) (*models.InputConfig, e
 // models.OutputConfig to be inserted into models.RunningInput
 // Note: error exists in the return for future calls that might require error
 func (c *Config) buildOutput(name string, tbl *ast.Table) (*models.OutputConfig, error) {
-	filter, err := c.buildFilter(tbl)
+	filter, err := c.buildFilter("outputs."+name, tbl)
 	if err != nil {
 		return nil, err
 	}
@@ -1443,7 +1481,7 @@ func (c *Config) missingTomlField(_ reflect.Type, key string) error {
 	case "alias", "always_include_local_tags",
 		"collection_jitter", "collection_offset",
 		"data_format", "delay", "drop", "drop_original",
-		"fielddrop", "fieldpass", "flush_interval", "flush_jitter",
+		"fielddrop", "fieldexclude", "fieldinclude", "fieldpass", "flush_interval", "flush_jitter",
 		"grace",
 		"interval",
 		"lvm", // What is this used for?

config/config_test.go

@@ -76,10 +76,10 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) {
 # is unique`
 
 	filter := models.Filter{
-		NameDrop:  []string{"metricname2"},
-		NamePass:  []string{"metricname1", "ip_192.168.1.1_name"},
-		FieldDrop: []string{"other", "stuff"},
-		FieldPass: []string{"some", "strings"},
+		NameDrop:     []string{"metricname2"},
+		NamePass:     []string{"metricname1", "ip_192.168.1.1_name"},
+		FieldExclude: []string{"other", "stuff"},
+		FieldInclude: []string{"some", "strings"},
 		TagDropFilters: []models.TagFilter{
 			{
 				Name:   "badtag",
@@ -117,10 +117,10 @@ func TestConfig_LoadSingleInput(t *testing.T) {
 	input.Servers = []string{"localhost"}
 
 	filter := models.Filter{
-		NameDrop:  []string{"metricname2"},
-		NamePass:  []string{"metricname1"},
-		FieldDrop: []string{"other", "stuff"},
-		FieldPass: []string{"some", "strings"},
+		NameDrop:     []string{"metricname2"},
+		NamePass:     []string{"metricname1"},
+		FieldExclude: []string{"other", "stuff"},
+		FieldInclude: []string{"some", "strings"},
 		TagDropFilters: []models.TagFilter{
 			{
 				Name:   "badtag",
@@ -162,8 +162,8 @@ func TestConfig_LoadSingleInput_WithSeparators(t *testing.T) {
 		NameDropSeparators: ".",
 		NamePass:           []string{"metricname1"},
 		NamePassSeparators: ".",
-		FieldDrop:          []string{"other", "stuff"},
-		FieldPass:          []string{"some", "strings"},
+		FieldExclude:       []string{"other", "stuff"},
+		FieldInclude:       []string{"some", "strings"},
 		TagDropFilters: []models.TagFilter{
 			{
 				Name:   "badtag",
@@ -209,10 +209,10 @@ func TestConfig_LoadDirectory(t *testing.T) {
 	expectedPlugins[0].Servers = []string{"localhost"}
 
 	filterMockup := models.Filter{
-		NameDrop:  []string{"metricname2"},
-		NamePass:  []string{"metricname1"},
-		FieldDrop: []string{"other", "stuff"},
-		FieldPass: []string{"some", "strings"},
+		NameDrop:     []string{"metricname2"},
+		NamePass:     []string{"metricname1"},
+		FieldExclude: []string{"other", "stuff"},
+		FieldInclude: []string{"some", "strings"},
 		TagDropFilters: []models.TagFilter{
 			{
 				Name:   "badtag",
@@ -253,10 +253,10 @@ func TestConfig_LoadDirectory(t *testing.T) {
 	expectedPlugins[2].Servers = []string{"192.168.1.1"}
 
 	filterMemcached := models.Filter{
-		NameDrop:  []string{"metricname2"},
-		NamePass:  []string{"metricname1"},
-		FieldDrop: []string{"other", "stuff"},
-		FieldPass: []string{"some", "strings"},
+		NameDrop:     []string{"metricname2"},
+		NamePass:     []string{"metricname1"},
+		FieldExclude: []string{"other", "stuff"},
+		FieldInclude: []string{"some", "strings"},
 		TagDropFilters: []models.TagFilter{
 			{
 				Name:   "badtag",
@@ -355,6 +355,15 @@ func TestConfig_LoadSpecialTypes(t *testing.T) {
 	require.Equal(t, "/path/", strings.TrimRight(input.Paths[0], "\r\n"))
 }
 
+func TestConfig_DeprecatedFilters(t *testing.T) {
+	c := config.NewConfig()
+	require.NoError(t, c.LoadConfig("./testdata/deprecated_field_filter.toml"))
+
+	require.Len(t, c.Inputs, 1)
+	require.Equal(t, []string{"foo", "bar", "baz"}, c.Inputs[0].Config.Filter.FieldInclude)
+	require.Equal(t, []string{"foo", "bar", "baz"}, c.Inputs[0].Config.Filter.FieldExclude)
+}
+
 func TestConfig_FieldNotDefined(t *testing.T) {
 	tests := []struct {
 		name     string

config/testdata/deprecated_field_filter.toml

@@ -0,0 +1,8 @@
+[[inputs.file]]
+  pass = ["foo"]
+  fieldpass = ["bar"]
+  fieldinclude = ["baz"]
+
+  drop = ["foo"]
+  fielddrop = ["bar"]
+  fieldexclude = ["baz"]

config/testdata/single_plugin.toml

@@ -2,8 +2,8 @@
   servers = ["localhost"]
   namepass = ["metricname1"]
   namedrop = ["metricname2"]
-  fieldpass = ["some", "strings"]
-  fielddrop = ["other", "stuff"]
+  fieldinclude = ["some", "strings"]
+  fieldexclude = ["other", "stuff"]
   interval = "5s"
   [inputs.memcached.tagpass]
     goodtag = ["mytag"]

config/testdata/single_plugin_env_vars.toml

@@ -18,8 +18,8 @@
   servers = ["$MY_TEST_SERVER"]
   namepass = ["metricname1", "ip_${MY_TEST_SERVER}_name"] # this comment will be ignored as well
   namedrop = ["metricname2"]
-  fieldpass = ["some", "strings"]
-  fielddrop = ["other", "stuff"]
+  fieldinclude = ["some", "strings"]
+  fieldexclude = ["other", "stuff"]
   interval = "$TEST_INTERVAL"
    ##### this input is provided to test multiline strings
   command = """

config/testdata/single_plugin_with_separators.toml

@@ -4,8 +4,8 @@
   namepass_separator = "."
   namedrop = ["metricname2"]
   namedrop_separator = "."
-  fieldpass = ["some", "strings"]
-  fielddrop = ["other", "stuff"]
+  fieldinclude = ["some", "strings"]
+  fieldexclude = ["other", "stuff"]
   interval = "5s"
   [inputs.memcached.tagpass]
     goodtag = ["mytag"]

docs/CONFIGURATION.md

@@ -457,7 +457,7 @@ avoid measurement collisions when defining multiple plugins:
   percpu = true
   totalcpu = false
   name_override = "percpu_usage"
-  fielddrop = ["cpu_time*"]
+  fieldexclude = ["cpu_time*"]
 ```
 
 ### Output Plugins
@@ -582,7 +582,7 @@ the originals.
 
 ```toml
 [[inputs.system]]
-  fieldpass = ["load1"] # collects system load1 metric.
+  fieldinclude = ["load1"] # collects system load1 metric.
 
 [[aggregators.minmax]]
   period = "30s"        # send & clear the aggregate every 30s.
@@ -600,7 +600,7 @@ to the `namepass` parameter.
 [[inputs.swap]]
 
 [[inputs.system]]
-  fieldpass = ["load1"] # collects system load1 metric.
+  fieldinclude = ["load1"] # collects system load1 metric.
 
 [[aggregators.minmax]]
   period = "30s"        # send & clear the aggregate every 30s.
@@ -688,14 +688,14 @@ removed the metric is removed. Tags and fields are modified before a metric is
 passed to a processor, aggregator, or output plugin. When used with an input
 plugin the filter applies after the input runs.
 
-- **fieldpass**:
+- **fieldinclude**:
 An array of [glob pattern][] strings.  Only fields whose field key matches a
 pattern in this list are emitted.
 
-- **fielddrop**:
-The inverse of `fieldpass`.  Fields with a field key matching one of the
+- **fieldexclude**:
+The inverse of `fieldinclude`.  Fields with a field key matching one of the
 patterns will be discarded from the metric.  This is tested on metrics after
-they have passed the `fieldpass` test.
+they have passed the `fieldinclude` test.
 
 - **taginclude**:
 An array of [glob pattern][] strings.  Only tags with a tag key matching one of
@@ -717,7 +717,7 @@ tags and the agent `host` tag.
 [[inputs.cpu]]
   percpu = true
   totalcpu = false
-  fielddrop = ["cpu_time"]
+  fieldexclude = ["cpu_time"]
   # Don't collect CPU data for cpu6 & cpu7
   [inputs.cpu.tagdrop]
     cpu = [ "cpu6", "cpu7" ]
@@ -746,18 +746,18 @@ tags and the agent `host` tag.
     instance = ["isatap*", "Local*"]
 ```
 
-#### Using fieldpass and fielddrop
+#### Using fieldinclude and fieldexclude
 
 ```toml
 # Drop all metrics for guest & steal CPU usage
 [[inputs.cpu]]
   percpu = false
   totalcpu = true
-  fielddrop = ["usage_guest", "usage_steal"]
+  fieldexclude = ["usage_guest", "usage_steal"]
 
 # Only store inode related metrics for disks
 [[inputs.disk]]
-  fieldpass = ["inodes*"]
+  fieldinclude = ["inodes*"]
 ```
 
 #### Using namepass and namedrop

docs/developers/DEPRECATION.md

@@ -89,5 +89,5 @@ Add filtering to the sample config, leave it commented out.
 ```toml
 [[inputs.system]]
   ## Uncomment to remove deprecated metrics.
-  # fielddrop = ["uptime_format"]
+  # fieldexclude = ["uptime_format"]
 ```

migrations/common/filter_options.go

@@ -5,8 +5,10 @@ type FilterOptions struct {
 	NameDrop       []string            `toml:"namedrop,omitempty"`
 	FieldPassOld   []string            `toml:"pass,omitempty"`
 	FieldPass      []string            `toml:"fieldpass,omitempty"`
+	FieldInclude   []string            `toml:"fieldinclude,omitempty"`
 	FieldDropOld   []string            `toml:"drop,omitempty"`
 	FieldDrop      []string            `toml:"fielddrop,omitempty"`
+	FieldExclude   []string            `toml:"fieldexclude,omitempty"`
 	TagPassFilters map[string][]string `toml:"tagpass,omitempty"`
 	TagDropFilters map[string][]string `toml:"tagdrop,omitempty"`
 	TagExclude     []string            `toml:"tagexclude,omitempty"`

migrations/common/input_options.go

@@ -17,11 +17,27 @@ type InputOptions struct {
 	NameDrop       []string            `toml:"namedrop,omitempty"`
 	FieldPassOld   []string            `toml:"pass,omitempty"`
 	FieldPass      []string            `toml:"fieldpass,omitempty"`
+	FieldInclude   []string            `toml:"fieldinclude,omitempty"`
 	FieldDropOld   []string            `toml:"drop,omitempty"`
 	FieldDrop      []string            `toml:"fielddrop,omitempty"`
+	FieldExclude   []string            `toml:"fieldexclude,omitempty"`
 	TagPassFilters map[string][]string `toml:"tagpass,omitempty"`
 	TagDropFilters map[string][]string `toml:"tagdrop,omitempty"`
 	TagExclude     []string            `toml:"tagexclude,omitempty"`
 	TagInclude     []string            `toml:"taginclude,omitempty"`
 	MetricPass     string              `toml:"metricpass,omitempty"`
 }
+
+func (io *InputOptions) Migrate() {
+	io.FieldInclude = append(io.FieldInclude, io.FieldPassOld...)
+	io.FieldInclude = append(io.FieldInclude, io.FieldPass...)
+
+	io.FieldPassOld = nil
+	io.FieldPass = nil
+
+	io.FieldExclude = append(io.FieldExclude, io.FieldDropOld...)
+	io.FieldExclude = append(io.FieldExclude, io.FieldDrop...)
+
+	io.FieldDropOld = nil
+	io.FieldDrop = nil
+}

migrations/common/output_options.go

@@ -9,11 +9,27 @@ type OutputOptions struct {
 	NameDrop       []string            `toml:"namedrop,omitempty"`
 	FieldPassOld   []string            `toml:"pass,omitempty"`
 	FieldPass      []string            `toml:"fieldpass,omitempty"`
+	FieldInclude   []string            `toml:"fieldinclude,omitempty"`
 	FieldDropOld   []string            `toml:"drop,omitempty"`
 	FieldDrop      []string            `toml:"fielddrop,omitempty"`
+	FieldExclude   []string            `toml:"fieldexclude,omitempty"`
 	TagPassFilters map[string][]string `toml:"tagpass,omitempty"`
 	TagDropFilters map[string][]string `toml:"tagdrop,omitempty"`
 	TagExclude     []string            `toml:"tagexclude,omitempty"`
 	TagInclude     []string            `toml:"taginclude,omitempty"`
 	MetricPass     string              `toml:"metricpass,omitempty"`
 }
+
+func (oo *OutputOptions) Migrate() {
+	oo.FieldInclude = append(oo.FieldInclude, oo.FieldPassOld...)
+	oo.FieldInclude = append(oo.FieldInclude, oo.FieldPass...)
+
+	oo.FieldPassOld = nil
+	oo.FieldPass = nil
+
+	oo.FieldExclude = append(oo.FieldExclude, oo.FieldDropOld...)
+	oo.FieldExclude = append(oo.FieldExclude, oo.FieldDrop...)
+
+	oo.FieldDropOld = nil
+	oo.FieldDrop = nil
+}