filter: add total processing volume metrics

Instead of just having metrics for number of records added/dropped, we now also have metrics for the total number of records and bytes processed by each filter. Signed-off-by: Dennis Kaarsemaker <dennis@kaarsemaker.net> Signed-off-by: root <root@sumit-acs.novalocal>
sumitd2 · Feb 8, 2023 · 2319923 · 2319923
1 parent a533df7
commit 2319923
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 0 deletions.
diff --git a/include/fluent-bit/flb_filter.h b/include/fluent-bit/flb_filter.h
@@ -87,6 +87,8 @@ struct flb_filter_instance {
      * --------
      */
     struct cmt *cmt;                      /* parent context               */
+    struct cmt_counter *cmt_records;      /* m: filter_records_total      */
+    struct cmt_counter *cmt_bytes;        /* m: filter_bytes_total        */
     struct cmt_counter *cmt_add_records;  /* m: filter_add_records_total  */
     struct cmt_counter *cmt_drop_records; /* m: filter_drop_records_total */
 

diff --git a/src/flb_filter.c b/src/flb_filter.c
@@ -152,6 +152,14 @@ void flb_filter_do(struct flb_input_chunk *ic,
 
 #ifdef FLB_HAVE_METRICS
             name = (char *) flb_filter_name(f_ins);
+
+            cmt_counter_add(f_ins->cmt_records, ts, in_records,
+                    1, (char *[]) {name});
+            cmt_counter_add(f_ins->cmt_bytes, ts, content_size,
+                    1, (char *[]) {name});
+
+            flb_metrics_sum(FLB_METRIC_N_RECORDS, in_records, f_ins->metrics);
+            flb_metrics_sum(FLB_METRIC_N_BYTES, content_size, f_ins->metrics);
 #endif
 
             /* Override buffer just if it was modified */
@@ -442,6 +450,22 @@ int flb_filter_init_all(struct flb_config *config)
             return -1;
         }
 
+        /* Register generic filter plugin metrics */
+        ins->cmt_records = cmt_counter_create(ins->cmt,
+                                                  "fluentbit", "filter",
+                                                  "records_total",
+                                                  "Total number of new records processed.",
+                                                  1, (char *[]) {"name"});
+        cmt_counter_set(ins->cmt_records, ts, 0, 1, (char *[]) {name});
+
+        /* Register generic filter plugin metrics */
+        ins->cmt_bytes = cmt_counter_create(ins->cmt,
+                                                  "fluentbit", "filter",
+                                                  "bytes_total",
+                                                  "Total number of new bytes processed.",
+                                                  1, (char *[]) {"name"});
+        cmt_counter_set(ins->cmt_bytes, ts, 0, 1, (char *[]) {name});
+
         /* Register generic filter plugin metrics */
         ins->cmt_add_records = cmt_counter_create(ins->cmt,
                                                   "fluentbit", "filter",
@@ -474,6 +498,8 @@ int flb_filter_init_all(struct flb_config *config)
         /* Register filter metrics */
         flb_metrics_add(FLB_METRIC_N_DROPPED, "drop_records", ins->metrics);
         flb_metrics_add(FLB_METRIC_N_ADDED, "add_records", ins->metrics);
+        flb_metrics_add(FLB_METRIC_N_RECORDS, "records", ins->metrics);
+        flb_metrics_add(FLB_METRIC_N_BYTES, "bytes", ins->metrics);
 #endif
 
         /*