Skip to content

Commit 02d30cb

Browse files
fstabfstab
committed
logfile as label
1 parent 0bc7019 commit 02d30cb

File tree

8 files changed

+218
-72
lines changed

8 files changed

+218
-72
lines changed

‎example/config.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ metrics:
1414
help: Total number of rejected recipients, partitioned by error message.
1515
match: '%{EXIM_DATE} %{EXIM_REMOTE_HOST} F=<%{EMAILADDRESS}> rejected RCPT <%{EMAILADDRESS}>: %{EXIM_MESSAGE:message}'
1616
labels:
17+
logfile: '{{base .logfile}}'
1718
error_message: '{{.message}}'
1819
server:
1920
host: localhost

‎exporter/grok.go‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,33 +36,39 @@ func Compile(pattern string, patterns *Patterns) (*oniguruma.Regex, error) {
3636
return result, nil
3737
}
3838

39-
func VerifyFieldNames(m *v2.MetricConfig, regex, deleteRegex *oniguruma.Regex) error {
39+
func VerifyFieldNames(m *v2.MetricConfig, regex, deleteRegex *oniguruma.Regex, additionalFieldDefinitions map[string]string) error {
4040
for _, template := range m.LabelTemplates {
41-
err := verifyFieldName(m.Name, template, regex)
41+
err := verifyFieldName(m.Name, template, regex, additionalFieldDefinitions)
4242
if err != nil {
4343
return err
4444
}
4545
}
4646
for _, template := range m.DeleteLabelTemplates {
47-
err := verifyFieldName(m.Name, template, deleteRegex)
47+
err := verifyFieldName(m.Name, template, deleteRegex, additionalFieldDefinitions)
4848
if err != nil {
4949
return err
5050
}
5151
}
5252
if m.ValueTemplate != nil {
53-
err := verifyFieldName(m.Name, m.ValueTemplate, regex)
53+
err := verifyFieldName(m.Name, m.ValueTemplate, regex, additionalFieldDefinitions)
5454
if err != nil {
5555
return err
5656
}
5757
}
5858
return nil
5959
}
6060

61-
func verifyFieldName(metricName string, template template.Template, regex *oniguruma.Regex) error {
61+
func verifyFieldName(metricName string, template template.Template, regex *oniguruma.Regex, additionalFieldDefinitions map[string]string) error {
6262
if template != nil {
6363
for _, grokFieldName := range template.ReferencedGrokFields() {
64-
if !regex.HasCaptureGroup(grokFieldName) {
65-
return fmt.Errorf("%v: grok field %v not found in match pattern", metricName, grokFieldName)
64+
if description, ok := additionalFieldDefinitions[grokFieldName]; ok {
65+
if regex.HasCaptureGroup(grokFieldName) {
66+
return fmt.Errorf("%v: field name %v is ambigous, as this field is defined in the grok pattern but is also a global field provided by grok_exporter for the %v", metricName, grokFieldName, description)
67+
}
68+
} else {
69+
if !regex.HasCaptureGroup(grokFieldName) {
70+
return fmt.Errorf("%v: grok field %v not found in match pattern", metricName, grokFieldName)
71+
}
6672
}
6773
}
6874
}

‎exporter/grok_test.go‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ import (
2222
"testing"
2323
)
2424

25+
// copy-and-paste from grok_exporter.go
26+
var additionalFieldDefinitions = map[string]string{
27+
"logfile": "full path of the log file",
28+
}
29+
2530
func TestGrok(t *testing.T) {
2631
patterns := loadPatternDir(t)
2732
t.Run("compile all patterns", func(t *testing.T) {
@@ -36,6 +41,9 @@ func TestGrok(t *testing.T) {
3641
t.Run("verify capture group", func(t *testing.T) {
3742
testVerifyCaptureGroup(t, patterns)
3843
})
44+
t.Run("verify field names", func(t *testing.T) {
45+
testVerifyFieldNames(t, patterns)
46+
})
3947
}
4048

4149
func testCompileAllPatterns(t *testing.T, patterns *Patterns) {
@@ -87,6 +95,25 @@ func testVerifyCaptureGroup(t *testing.T, patterns *Patterns) {
8795
regex.Free()
8896
}
8997

98+
func testVerifyFieldNames(t *testing.T, patterns *Patterns) {
99+
regex, err := Compile("log file %{WORD:logfile} user %{USER:user}", patterns)
100+
if err != nil {
101+
t.Fatal(err)
102+
}
103+
// .logfile not used in any label -> ok
104+
expectOK(t, regex, `
105+
name: text
106+
labels:
107+
user: '{{.user}}'`)
108+
// .logfile used in label -> error, ambiguous field
109+
expectError(t, regex, `
110+
name: text
111+
labels:
112+
logfile: '{{base .logfile}}'
113+
user: '{{.user}}'`)
114+
regex.Free()
115+
}
116+
90117
func expectOK(t *testing.T, regex *oniguruma.Regex, config string) {
91118
expect(t, regex, config, false)
92119
}
@@ -105,7 +132,7 @@ func expect(t *testing.T, regex *oniguruma.Regex, config string, isErrorExpected
105132
if err != nil {
106133
t.Fatal(err)
107134
}
108-
err = VerifyFieldNames(cfg, regex, nil)
135+
err = VerifyFieldNames(cfg, regex, nil, additionalFieldDefinitions)
109136
if isErrorExpected && err == nil {
110137
t.Fatal("Expected error, but got no error.")
111138
}

‎exporter/metrics.go‎

Lines changed: 54 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,9 @@ type Metric interface {
3434
Collector() prometheus.Collector
3535

3636
// Returns the match if the line matched, and nil if the line didn't match.
37-
ProcessMatch(line string) (*Match, error)
37+
ProcessMatch(line string, additionalFields map[string]string) (*Match, error)
3838
// Returns the match if the delete pattern matched, nil otherwise.
39-
ProcessDeleteMatch(line string) (*Match, error)
39+
ProcessDeleteMatch(line string, additionalFields map[string]string) (*Match, error)
4040
// Remove old metrics
4141
ProcessRetention() error
4242
}
@@ -171,7 +171,7 @@ func (m *observeMetric) processMatch(line string, cb func(value float64)) (*Matc
171171
}
172172
defer searchResult.Free()
173173
if searchResult.IsMatch() {
174-
floatVal, err := floatValue(m.Name(), searchResult, m.valueTemplate)
174+
floatVal, err := floatValue(m.Name(), searchResult, m.valueTemplate, nil)
175175
if err != nil {
176176
return nil, err
177177
}
@@ -184,19 +184,19 @@ func (m *observeMetric) processMatch(line string, cb func(value float64)) (*Matc
184184
}
185185
}
186186

187-
func (m *metricWithLabels) processMatch(line string, cb func(labels map[string]string)) (*Match, error) {
187+
func (m *metricWithLabels) processMatch(line string, additionalFields map[string]string, callback func(labels map[string]string)) (*Match, error) {
188188
searchResult, err := m.regex.Search(line)
189189
if err != nil {
190190
return nil, fmt.Errorf("error while processing metric %v: %v", m.Name(), err.Error())
191191
}
192192
defer searchResult.Free()
193193
if searchResult.IsMatch() {
194-
labels, err := labelValues(m.Name(), searchResult, m.labelTemplates)
194+
labels, err := labelValues(m.Name(), searchResult, m.labelTemplates, additionalFields)
195195
if err != nil {
196196
return nil, err
197197
}
198198
m.labelValueTracker.Observe(labels)
199-
cb(labels)
199+
callback(labels)
200200
return &Match{
201201
Value: 1.0,
202202
Labels: labels,
@@ -206,23 +206,23 @@ func (m *metricWithLabels) processMatch(line string, cb func(labels map[string]s
206206
}
207207
}
208208

209-
func (m *observeMetricWithLabels) processMatch(line string, cb func(value float64, labels map[string]string)) (*Match, error) {
209+
func (m *observeMetricWithLabels) processMatch(line string, additionalFields map[string]string, callback func(value float64, labels map[string]string)) (*Match, error) {
210210
searchResult, err := m.regex.Search(line)
211211
if err != nil {
212212
return nil, fmt.Errorf("error processing metric %v: %v", m.Name(), err.Error())
213213
}
214214
defer searchResult.Free()
215215
if searchResult.IsMatch() {
216-
floatVal, err := floatValue(m.Name(), searchResult, m.valueTemplate)
216+
floatVal, err := floatValue(m.Name(), searchResult, m.valueTemplate, additionalFields)
217217
if err != nil {
218218
return nil, err
219219
}
220-
labels, err := labelValues(m.Name(), searchResult, m.labelTemplates)
220+
labels, err := labelValues(m.Name(), searchResult, m.labelTemplates, additionalFields)
221221
if err != nil {
222222
return nil, err
223223
}
224224
m.labelValueTracker.Observe(labels)
225-
cb(floatVal, labels)
225+
callback(floatVal, labels)
226226
return &Match{
227227
Value: floatVal,
228228
Labels: labels,
@@ -232,7 +232,7 @@ func (m *observeMetricWithLabels) processMatch(line string, cb func(value float6
232232
}
233233
}
234234

235-
func (m *metric) ProcessDeleteMatch(line string) (*Match, error) {
235+
func (m *metric) ProcessDeleteMatch(line string, additionalFields map[string]string) (*Match, error) {
236236
if m.deleteRegex == nil {
237237
return nil, nil
238238
}
@@ -246,7 +246,7 @@ func (m *metric) ProcessRetention() error {
246246
return fmt.Errorf("error processing metric %v: retention is currently only supported for metrics with labels.", m.Name())
247247
}
248248

249-
func (m *metricWithLabels) processDeleteMatch(line string, vec deleterMetric) (*Match, error) {
249+
func (m *metricWithLabels) processDeleteMatch(line string, vec deleterMetric, additionalFields map[string]string) (*Match, error) {
250250
if m.deleteRegex == nil {
251251
return nil, nil
252252
}
@@ -256,7 +256,7 @@ func (m *metricWithLabels) processDeleteMatch(line string, vec deleterMetric) (*
256256
}
257257
defer searchResult.Free()
258258
if searchResult.IsMatch() {
259-
deleteLabels, err := labelValues(m.Name(), searchResult, m.deleteLabelTemplates)
259+
deleteLabels, err := labelValues(m.Name(), searchResult, m.deleteLabelTemplates, additionalFields)
260260
if err != nil {
261261
return nil, err
262262
}
@@ -284,27 +284,27 @@ func (m *metricWithLabels) processRetention(vec deleterMetric) error {
284284
return nil
285285
}
286286

287-
func (m *counterMetric) ProcessMatch(line string) (*Match, error) {
287+
func (m *counterMetric) ProcessMatch(line string, additionalFields map[string]string) (*Match, error) {
288288
return m.processMatch(line, func() {
289289
m.counter.Inc()
290290
})
291291
}
292292

293-
func (m *counterVecMetric) ProcessMatch(line string) (*Match, error) {
294-
return m.processMatch(line, func(labels map[string]string) {
293+
func (m *counterVecMetric) ProcessMatch(line string, additionalFields map[string]string) (*Match, error) {
294+
return m.processMatch(line, additionalFields, func(labels map[string]string) {
295295
m.counterVec.With(labels).Inc()
296296
})
297297
}
298298

299-
func (m *counterVecMetric) ProcessDeleteMatch(line string) (*Match, error) {
300-
return m.processDeleteMatch(line, m.counterVec)
299+
func (m *counterVecMetric) ProcessDeleteMatch(line string, additionalFields map[string]string) (*Match, error) {
300+
return m.processDeleteMatch(line, m.counterVec, additionalFields)
301301
}
302302

303303
func (m *counterVecMetric) ProcessRetention() error {
304304
return m.processRetention(m.counterVec)
305305
}
306306

307-
func (m *gaugeMetric) ProcessMatch(line string) (*Match, error) {
307+
func (m *gaugeMetric) ProcessMatch(line string, additionalFields map[string]string) (*Match, error) {
308308
return m.processMatch(line, func(value float64) {
309309
if m.cumulative {
310310
m.gauge.Add(value)
@@ -314,8 +314,8 @@ func (m *gaugeMetric) ProcessMatch(line string) (*Match, error) {
314314
})
315315
}
316316

317-
func (m *gaugeVecMetric) ProcessMatch(line string) (*Match, error) {
318-
return m.processMatch(line, func(value float64, labels map[string]string) {
317+
func (m *gaugeVecMetric) ProcessMatch(line string, additionalFields map[string]string) (*Match, error) {
318+
return m.processMatch(line, additionalFields, func(value float64, labels map[string]string) {
319319
if m.cumulative {
320320
m.gaugeVec.With(labels).Add(value)
321321
} else {
@@ -324,48 +324,48 @@ func (m *gaugeVecMetric) ProcessMatch(line string) (*Match, error) {
324324
})
325325
}
326326

327-
func (m *gaugeVecMetric) ProcessDeleteMatch(line string) (*Match, error) {
328-
return m.processDeleteMatch(line, m.gaugeVec)
327+
func (m *gaugeVecMetric) ProcessDeleteMatch(line string, additionalFields map[string]string) (*Match, error) {
328+
return m.processDeleteMatch(line, m.gaugeVec, additionalFields)
329329
}
330330

331331
func (m *gaugeVecMetric) ProcessRetention() error {
332332
return m.processRetention(m.gaugeVec)
333333
}
334334

335-
func (m *histogramMetric) ProcessMatch(line string) (*Match, error) {
335+
func (m *histogramMetric) ProcessMatch(line string, additionalFields map[string]string) (*Match, error) {
336336
return m.processMatch(line, func(value float64) {
337337
m.histogram.Observe(value)
338338
})
339339
}
340340

341-
func (m *histogramVecMetric) ProcessMatch(line string) (*Match, error) {
342-
return m.processMatch(line, func(value float64, labels map[string]string) {
341+
func (m *histogramVecMetric) ProcessMatch(line string, additionalFields map[string]string) (*Match, error) {
342+
return m.processMatch(line, additionalFields, func(value float64, labels map[string]string) {
343343
m.histogramVec.With(labels).Observe(value)
344344
})
345345
}
346346

347-
func (m *histogramVecMetric) ProcessDeleteMatch(line string) (*Match, error) {
348-
return m.processDeleteMatch(line, m.histogramVec)
347+
func (m *histogramVecMetric) ProcessDeleteMatch(line string, additionalFields map[string]string) (*Match, error) {
348+
return m.processDeleteMatch(line, m.histogramVec, additionalFields)
349349
}
350350

351351
func (m *histogramVecMetric) ProcessRetention() error {
352352
return m.processRetention(m.histogramVec)
353353
}
354354

355-
func (m *summaryMetric) ProcessMatch(line string) (*Match, error) {
355+
func (m *summaryMetric) ProcessMatch(line string, additionalFields map[string]string) (*Match, error) {
356356
return m.processMatch(line, func(value float64) {
357357
m.summary.Observe(value)
358358
})
359359
}
360360

361-
func (m *summaryVecMetric) ProcessMatch(line string) (*Match, error) {
362-
return m.processMatch(line, func(value float64, labels map[string]string) {
361+
func (m *summaryVecMetric) ProcessMatch(line string, additionalFields map[string]string) (*Match, error) {
362+
return m.processMatch(line, additionalFields, func(value float64, labels map[string]string) {
363363
m.summaryVec.With(labels).Observe(value)
364364
})
365365
}
366366

367-
func (m *summaryVecMetric) ProcessDeleteMatch(line string) (*Match, error) {
368-
return m.processDeleteMatch(line, m.summaryVec)
367+
func (m *summaryVecMetric) ProcessDeleteMatch(line string, additionalFields map[string]string) (*Match, error) {
368+
return m.processDeleteMatch(line, m.summaryVec, additionalFields)
369369
}
370370

371371
func (m *summaryVecMetric) ProcessRetention() error {
@@ -484,10 +484,10 @@ func NewSummaryMetric(cfg *configuration.MetricConfig, regex *oniguruma.Regex, d
484484
}
485485
}
486486

487-
func labelValues(metricName string, searchResult *oniguruma.SearchResult, templates []template.Template) (map[string]string, error) {
487+
func labelValues(metricName string, searchResult *oniguruma.SearchResult, templates []template.Template, additionalFields map[string]string) (map[string]string, error) {
488488
result := make(map[string]string, len(templates))
489489
for _, t := range templates {
490-
value, err := evalTemplate(searchResult, t)
490+
value, err := evalTemplate(searchResult, t, additionalFields)
491491
if err != nil {
492492
return nil, fmt.Errorf("error processing metric %v: %v", metricName, err.Error())
493493
}
@@ -496,28 +496,36 @@ func labelValues(metricName string, searchResult *oniguruma.SearchResult, templa
496496
return result, nil
497497
}
498498

499-
func floatValue(metricName string, searchResult *oniguruma.SearchResult, valueTemplate template.Template) (float64, error) {
500-
stringVal, err := evalTemplate(searchResult, valueTemplate)
499+
func floatValue(metricName string, searchResult *oniguruma.SearchResult, valueTemplate template.Template, additionalFields map[string]string) (float64, error) {
500+
stringVal, err := evalTemplate(searchResult, valueTemplate, additionalFields)
501501
if err != nil {
502502
return 0, fmt.Errorf("error processing metric %v: %v", metricName, err.Error())
503503
}
504504
floatVal, err := strconv.ParseFloat(stringVal, 64)
505505
if err != nil {
506-
return 0, fmt.Errorf("error processing metric %v: value matches '%v', which is not a valid number.", metricName, stringVal)
506+
return 0, fmt.Errorf("error processing metric %v: value matches '%v', which is not a valid number", metricName, stringVal)
507507
}
508508
return floatVal, nil
509509
}
510510

511-
func evalTemplate(searchResult *oniguruma.SearchResult, t template.Template) (string, error) {
512-
grokValues := make(map[string]string, len(t.ReferencedGrokFields()))
513-
for _, field := range t.ReferencedGrokFields() {
514-
value, err := searchResult.GetCaptureGroupByName(field)
515-
if err != nil {
516-
return "", err
511+
func evalTemplate(searchResult *oniguruma.SearchResult, t template.Template, additionalFields map[string]string) (string, error) {
512+
var (
513+
values = make(map[string]string, len(t.ReferencedGrokFields()))
514+
value string
515+
ok bool
516+
err error
517+
field string
518+
)
519+
for _, field = range t.ReferencedGrokFields() {
520+
if value, ok = additionalFields[field]; !ok {
521+
value, err = searchResult.GetCaptureGroupByName(field)
522+
if err != nil {
523+
return "", err
524+
}
517525
}
518-
grokValues[field] = value
526+
values[field] = value
519527
}
520-
return t.Execute(grokValues)
528+
return t.Execute(values)
521529
}
522530

523531
func prometheusLabels(templates []template.Template) []string {

0 commit comments

Comments
 (0)