feat(agent): Allow separators for namepass and namedrop filters (infl…

…uxdata#14361)

config/config.go

@@ -1331,7 +1331,9 @@ func (c *Config) buildFilter(tbl *ast.Table) (models.Filter, error) {
 	f := models.Filter{}
 
 	c.getFieldStringSlice(tbl, "namepass", &f.NamePass)
+	c.getFieldString(tbl, "namepass_separator", &f.NamePassSeparators)
 	c.getFieldStringSlice(tbl, "namedrop", &f.NameDrop)
+	c.getFieldString(tbl, "namedrop_separator", &f.NameDropSeparators)
 
 	c.getFieldStringSlice(tbl, "pass", &f.FieldPass)
 	c.getFieldStringSlice(tbl, "fieldpass", &f.FieldPass)
@@ -1446,7 +1448,7 @@ func (c *Config) missingTomlField(_ reflect.Type, key string) error {
 		"interval",
 		"lvm", // What is this used for?
 		"metric_batch_size", "metric_buffer_limit", "metricpass",
-		"name_override", "name_prefix", "name_suffix", "namedrop", "namepass",
+		"name_override", "name_prefix", "name_suffix", "namedrop", "namedrop_separator", "namepass", "namepass_separator",
 		"order",
 		"pass", "period", "precision",
 		"tagdrop", "tagexclude", "taginclude", "tagpass", "tags":

config/config_test.go

@@ -150,6 +150,49 @@ func TestConfig_LoadSingleInput(t *testing.T) {
 	require.Equal(t, inputConfig, c.Inputs[0].Config, "Testdata did not produce correct memcached metadata.")
 }
 
+func TestConfig_LoadSingleInput_WithSeparators(t *testing.T) {
+	c := config.NewConfig()
+	require.NoError(t, c.LoadConfig("./testdata/single_plugin_with_separators.toml"))
+
+	input := inputs.Inputs["memcached"]().(*MockupInputPlugin)
+	input.Servers = []string{"localhost"}
+
+	filter := models.Filter{
+		NameDrop:           []string{"metricname2"},
+		NameDropSeparators: ".",
+		NamePass:           []string{"metricname1"},
+		NamePassSeparators: ".",
+		FieldDrop:          []string{"other", "stuff"},
+		FieldPass:          []string{"some", "strings"},
+		TagDropFilters: []models.TagFilter{
+			{
+				Name:   "badtag",
+				Values: []string{"othertag"},
+			},
+		},
+		TagPassFilters: []models.TagFilter{
+			{
+				Name:   "goodtag",
+				Values: []string{"mytag"},
+			},
+		},
+	}
+	require.NoError(t, filter.Compile())
+	inputConfig := &models.InputConfig{
+		Name:     "memcached",
+		Filter:   filter,
+		Interval: 5 * time.Second,
+	}
+	inputConfig.Tags = make(map[string]string)
+
+	// Ignore Log, Parser and ID
+	c.Inputs[0].Input.(*MockupInputPlugin).Log = nil
+	c.Inputs[0].Input.(*MockupInputPlugin).parser = nil
+	c.Inputs[0].Config.ID = ""
+	require.Equal(t, input, c.Inputs[0].Input, "Testdata did not produce a correct memcached struct.")
+	require.Equal(t, inputConfig, c.Inputs[0].Config, "Testdata did not produce correct memcached metadata.")
+}
+
 func TestConfig_LoadDirectory(t *testing.T) {
 	c := config.NewConfig()
 

config/testdata/single_plugin_with_separators.toml

@@ -0,0 +1,13 @@
+[[inputs.memcached]]
+  servers = ["localhost"]
+  namepass = ["metricname1"]
+  namepass_separator = "."
+  namedrop = ["metricname2"]
+  namedrop_separator = "."
+  fieldpass = ["some", "strings"]
+  fielddrop = ["other", "stuff"]
+  interval = "5s"
+  [inputs.memcached.tagpass]
+    goodtag = ["mytag"]
+  [inputs.memcached.tagdrop]
+    badtag = ["othertag"]

docs/CONFIGURATION.md

@@ -626,11 +626,15 @@ sent onwards to the next stage of processing.
 
 - **namepass**:
 An array of [glob pattern][] strings. Only metrics whose measurement name
-matches a pattern in this list are emitted.
+matches a pattern in this list are emitted. Additionally, custom list of
+separators can be specified using `namepass_separator`. These separators
+are excluded from wildcard glob pattern matching.
 
 - **namedrop**:
 The inverse of `namepass`. If a match is found the metric is discarded. This
-is tested on metrics after they have passed the `namepass` test.
+is tested on metrics after they have passed the `namepass` test. Additionally,
+custom list of separators can be specified using `namedrop_separator`. These
+separators are excluded from wildcard glob pattern matching.
 
 - **tagpass**:
 A table mapping tag keys to arrays of [glob pattern][] strings.  Only metrics
@@ -770,6 +774,26 @@ tags and the agent `host` tag.
   namepass = ["rest_client_*"]
 ```
 
+#### Using namepass and namedrop with separators
+
+```toml
+# Pass all metrics of type 'A.C.B' and drop all others like 'A.C.D.B'
+[[inputs.socket_listener]]
+  data_format = "graphite"
+  templates = ["measurement*"]
+
+  namepass = ["A.*.B"]
+  namepass_separator = "."
+
+# Drop all metrics of type 'A.C.B' and pass all others like 'A.C.D.B'
+[[inputs.socket_listener]]
+  data_format = "graphite"
+  templates = ["measurement*"]
+
+  namedrop = ["A.*.B"]
+  namedrop_separator = "."
+```
+
 #### Using taginclude and tagexclude
 
 ```toml

filter/filter.go

@@ -12,20 +12,29 @@ type Filter interface {
 
 // Compile takes a list of string filters and returns a Filter interface
 // for matching a given string against the filter list. The filter list
-// supports glob matching too, ie:
+// supports glob matching with separators too, ie:
 //
 //	f, _ := Compile([]string{"cpu", "mem", "net*"})
 //	f.Match("cpu")     // true
 //	f.Match("network") // true
 //	f.Match("memory")  // false
-func Compile(filters []string) (Filter, error) {
+//
+// separators are only to be used for globbing filters, ie:
+//
+//	f, _ := Compile([]string{"cpu.*.count"}, '.')
+//	f.Match("cpu.count")     // false
+//	f.Match("cpu.measurement.count") // true
+//	f.Match("cpu.field.measurement.count")  // false
+//
+// Compile will return nil if the filter list is empty.
+func Compile(filters []string, separators ...rune) (Filter, error) {
 	// return if there is nothing to compile
 	if len(filters) == 0 {
 		return nil, nil
 	}
 
 	// check if we can compile a non-glob filter
-	noGlob := true
+	noGlob := len(separators) == 0
 	for _, filter := range filters {
 		if hasMeta(filter) {
 			noGlob = false
@@ -38,14 +47,14 @@ func Compile(filters []string) (Filter, error) {
 		// return non-globbing filter if not needed.
 		return compileFilterNoGlob(filters), nil
 	case len(filters) == 1:
-		return glob.Compile(filters[0])
+		return glob.Compile(filters[0], separators...)
 	default:
-		return glob.Compile("{" + strings.Join(filters, ",") + "}")
+		return glob.Compile("{"+strings.Join(filters, ",")+"}", separators...)
 	}
 }
 
-func MustCompile(filters []string) Filter {
-	f, err := Compile(filters)
+func MustCompile(filters []string, separators ...rune) Filter {
+	f, err := Compile(filters, separators...)
 	if err != nil {
 		panic(err)
 	}

filter/filter_test.go

@@ -35,6 +35,18 @@ func TestCompile(t *testing.T) {
 	require.False(t, f.Match("cpu0"))
 	require.True(t, f.Match("mem"))
 	require.True(t, f.Match("network"))
+
+	f, err = Compile([]string{"cpu.*.count"}, '.')
+	require.NoError(t, err)
+	require.False(t, f.Match("cpu.count"))
+	require.True(t, f.Match("cpu.measurement.count"))
+	require.False(t, f.Match("cpu.field.measurement.count"))
+
+	f, err = Compile([]string{"cpu.*.count"}, '.', ',')
+	require.NoError(t, err)
+	require.True(t, f.Match("cpu.measurement.count"))
+	require.False(t, f.Match("cpu.,.count")) // ',' is not considered under * as it is specified as a separator
+	require.False(t, f.Match("cpu.field,measurement.count"))
 }
 
 func TestIncludeExclude(t *testing.T) {

models/filter.go

@@ -33,10 +33,12 @@ func (tf *TagFilter) Compile() error {
 
 // Filter containing drop/pass and tagdrop/tagpass rules
 type Filter struct {
-	NameDrop       []string
-	nameDropFilter filter.Filter
-	NamePass       []string
-	namePassFilter filter.Filter
+	NameDrop           []string
+	NameDropSeparators string
+	nameDropFilter     filter.Filter
+	NamePass           []string
+	NamePassSeparators string
+	namePassFilter     filter.Filter
 
 	FieldDrop       []string
 	fieldDropFilter filter.Filter
@@ -78,11 +80,11 @@ func (f *Filter) Compile() error {
 
 	if f.selectActive {
 		var err error
-		f.nameDropFilter, err = filter.Compile(f.NameDrop)
+		f.nameDropFilter, err = filter.Compile(f.NameDrop, []rune(f.NameDropSeparators)...)
 		if err != nil {
 			return fmt.Errorf("error compiling 'namedrop', %w", err)
 		}
-		f.namePassFilter, err = filter.Compile(f.NamePass)
+		f.namePassFilter, err = filter.Compile(f.NamePass, []rune(f.NamePassSeparators)...)
 		if err != nil {
 			return fmt.Errorf("error compiling 'namepass', %w", err)
 		}

models/filter_test.go

@@ -146,6 +146,42 @@ func TestFilter_NamePass(t *testing.T) {
 	}
 }
 
+func TestFilter_NamePass_WithSeparator(t *testing.T) {
+	f := Filter{
+		NamePass:           []string{"foo.*.bar", "foo.*.abc.*.bar"},
+		NamePassSeparators: ".,",
+	}
+	require.NoError(t, f.Compile())
+
+	passes := []string{
+		"foo..bar",
+		"foo.abc.bar",
+		"foo..abc..bar",
+		"foo.xyz.abc.xyz-xyz.bar",
+	}
+
+	drops := []string{
+		"foo.bar",
+		"foo.abc,.bar", // "abc," is not considered under * as ',' is specified as a separator
+		"foo..abc.bar", // ".abc" shall not be matched under * as '.' is specified as a separator
+		"foo.abc.abc.bar",
+		"foo.xyz.abc.xyz.xyz.bar",
+		"foo.xyz.abc.xyz,xyz.bar",
+	}
+
+	for _, measurement := range passes {
+		if !f.shouldNamePass(measurement) {
+			t.Errorf("Expected measurement %s to pass", measurement)
+		}
+	}
+
+	for _, measurement := range drops {
+		if f.shouldNamePass(measurement) {
+			t.Errorf("Expected measurement %s to drop", measurement)
+		}
+	}
+}
+
 func TestFilter_NameDrop(t *testing.T) {
 	f := Filter{
 		NameDrop: []string{"foo*", "cpu_usage_idle"},
@@ -180,6 +216,42 @@ func TestFilter_NameDrop(t *testing.T) {
 	}
 }
 
+func TestFilter_NameDrop_WithSeparator(t *testing.T) {
+	f := Filter{
+		NameDrop:           []string{"foo.*.bar", "foo.*.abc.*.bar"},
+		NameDropSeparators: ".,",
+	}
+	require.NoError(t, f.Compile())
+
+	drops := []string{
+		"foo..bar",
+		"foo.abc.bar",
+		"foo..abc..bar",
+		"foo.xyz.abc.xyz-xyz.bar",
+	}
+
+	passes := []string{
+		"foo.bar",
+		"foo.abc,.bar", // "abc," is not considered under * as ',' is specified as a separator
+		"foo..abc.bar", // ".abc" shall not be matched under * as '.' is specified as a separator
+		"foo.abc.abc.bar",
+		"foo.xyz.abc.xyz.xyz.bar",
+		"foo.xyz.abc.xyz,xyz.bar",
+	}
+
+	for _, measurement := range passes {
+		if !f.shouldNamePass(measurement) {
+			t.Errorf("Expected measurement %s to pass", measurement)
+		}
+	}
+
+	for _, measurement := range drops {
+		if f.shouldNamePass(measurement) {
+			t.Errorf("Expected measurement %s to drop", measurement)
+		}
+	}
+}
+
 func TestFilter_FieldPass(t *testing.T) {
 	f := Filter{
 		FieldPass: []string{"foo*", "cpu_usage_idle"},