[yugabyte#19943] docdb: Add new regex filters to Prometheus metric endpoint

Summary:
This diff creates brand new API for YBA to filter prometheus metric output, and also enable server level aggregation for all tablet and table metrics when new API is used. To enable the new API, user need to explicit add `version=v2` to the URL.

**New URL parameters for version control:**
`version`: Options(“v1”, “v2”), Default(“v1”).

"v1" means endpoint expect old regex filters:`priority_regex` and `metrics`
"v2" means endpoint expect new regex filters, and ignore the v1 filters. Tablet and table metrics will be aggregated to server level by default.

**New regex filters:**
`server_allowlist`: Options( “ALL”, “NONE”, Customized regex), Default (“ALL”)
`server_blocklist`: Options( “ALL”, “NONE”, Customized regex), Default (“NONE”)
`table_allowlist`: Options( “ALL”, “NONE”, Customized regex), Default (“ALL”)
`table_blocklist`: Options( “ALL”, “NONE”, Customized regex), Default (“NONE”)

A metric produced internally is tablet or table type will be emitted at table level if and only if it matches table_allowlist and does not match table_blocklist.
A metric produced internally is tablet, table, or server type will be emitted at the server level if and only if it matches server_allowlist and does not match server_blocklist.

**To distinguish whether a metric from the output is on stream, table, or server level:**
```
If metric_attributes contains “stream_id” :
    Stream level
else if metric_attributes contains “table_id” :
    Table level
else :
    Server level
```

**Others:**
If we accidentally call a server without the new code using the new v2 version URL, the server will generate a huge list of metrics at the table level, which can consume large amount of memory. Thus, it is recommended to add `metrics=` in the v2 URL , so that the server will just return empty output.
URL Example: `/prometheus-metrics?version=v2&table_allowlist=<regex>&table_blocklist=<regex>&server_allowlist<regex>&server_blocklist=<regex>&metrics=`
Jira: DB-8907

Test Plan:
PrometheusMetricFilterTest.TestVOneDefault
PrometheusMetricFilterTest.TestVOnePriorityRegex
PrometheusMetricFilterTest.TestVTwoDefault
PrometheusMetricFilterTest.TestVTwoTableLevel
PrometheusMetricFilterTest.TestVTwoServerLevel

Also tested on portal

Reviewers: amitanand, esheng, rthallam, mlillibridge

Reviewed By: amitanand, esheng, mlillibridge

Subscribers: sanketh, amalyshev, nbhatia, ybase, bogdan

Differential Revision: https://phorge.dev.yugabyte.com/D31259

Author: yusong-yan

src/yb/server/default-path-handlers.cc

@@ -538,13 +538,43 @@ static void ParseRequestOptions(const Webserver::WebRequest& req,
   if (prometheus_opts) {
     ParseMetricOptions(req, prometheus_opts);
 
-    prometheus_opts->priority_regex_string = FindWithDefault(
-        req.parsed_args, "priority_regex", ".*");
-
     if (const std::string* arg_p = FindOrNull(req.parsed_args, "show_help")) {
       prometheus_opts->export_help_and_type =
           ExportHelpAndType(ParseLeadingBoolValue(arg_p->c_str(), false));
     }
+
+    prometheus_opts->version = FindWithDefault(req.parsed_args, "version",
+        kFilterVersionOne);
+
+    if (prometheus_opts->version == kFilterVersionTwo) {
+      // Set it to accept all metrics, because we ignore metrics URL parameter when using v2.
+      prometheus_opts->general_metrics_allowlist = std::nullopt;
+
+      auto FindHandlingAllOrNone = [&](
+          const std::string& arg, const std::string& default_value) -> std::string {
+        std::string regex_string = FindWithDefault(req.parsed_args, arg, default_value);
+        if (regex_string == "ALL") {
+          return ".*";
+        } else if (regex_string == "NONE") {
+          return "";
+        }
+        return regex_string;
+      };
+
+      prometheus_opts->table_allowlist_string = FindHandlingAllOrNone("table_allowlist", "ALL");
+
+      prometheus_opts->table_blocklist_string = FindHandlingAllOrNone("table_blocklist", "NONE");
+
+      prometheus_opts->server_allowlist_string = FindHandlingAllOrNone("server_allowlist", "ALL");
+
+      prometheus_opts->server_blocklist_string = FindHandlingAllOrNone("server_blocklist", "NONE");
+    } else {
+      prometheus_opts->priority_regex_string = FindWithDefault(
+          req.parsed_args, "priority_regex", ".*");
+      LOG_IF(WARNING, prometheus_opts->version != kFilterVersionOne)
+          << "Prometheus endpoint URL parameter version=" << prometheus_opts->version
+          << " is not recognized. Only v1 or v2 can be accepted.";
+    }
   }
 
   if (json_mode) {

src/yb/util/CMakeLists.txt

@@ -394,6 +394,7 @@ ADD_YB_TEST(os-util-test)
 ADD_YB_TEST(path_util-test)
 ADD_YB_TEST(priority_queue-test)
 ADD_YB_TEST(priority_thread_pool-test)
+ADD_YB_TEST(prometheus_metric_filter-test)
 ADD_YB_TEST(pstack_watcher-test)
 ADD_YB_TEST(random-test)
 ADD_YB_TEST(random_util-test)

src/yb/util/metric_entity.h

@@ -89,12 +89,25 @@ struct MetricJsonOptions : public MetricOptions {
 
 YB_STRONGLY_TYPED_BOOL(ExportHelpAndType);
 
+static const std::string kFilterVersionOne = "v1";
+static const std::string kFilterVersionTwo = "v2";
+
 struct MetricPrometheusOptions : public MetricOptions {
   // Include #TYPE and #HELP in Prometheus metrics output
   ExportHelpAndType export_help_and_type{ExportHelpAndType::kFalse};
 
-  // Metrics that shows on table level.
+  std::string version = kFilterVersionOne;
+
+  // For filtering table level metrics when version is equal to kFilterVersionOne.
   std::string priority_regex_string = ".*";
+
+  // The four regexs are for filtering table level and server level metrics
+  // when version is equal to kFilterVersionTwo.
+  std::string table_allowlist_string = ".*";
+  std::string table_blocklist_string = "";
+
+  std::string server_allowlist_string = ".*";
+  std::string server_blocklist_string = "";
 };
 
 class MetricEntityPrototype {

src/yb/util/metrics-test.cc

@@ -678,60 +678,4 @@ TEST_F(MetricsTest, VerifyHelpAndTypeTags) {
       "# HELP t_lag Test lag description\n# TYPE t_lag gauge"));
 }
 
-TEST_F(MetricsTest, PrometheusMetricFilterTest) {
-  MetricEntity::AttributeMap entity_attr;
-  entity_attr["tablet_id"] = "tablet_id_42";
-  entity_attr["table_name"] = "test_table";
-  entity_attr["table_id"] = "table_id_43";
-  scoped_refptr<MetricEntity> tablet_entity =
-      METRIC_ENTITY_tablet.Instantiate(&registry_, "tablet_entity_id", entity_attr);
-  // Initilize two tablet metrics.
-  scoped_refptr<Gauge> gauge = METRIC_t_gauge.Instantiate(tablet_entity, 0);
-  scoped_refptr<Counter> counter = METRIC_t_counter.Instantiate(tablet_entity);
-  std::string kGaugeName = "t_gauge";
-  std::string kCounterName = "t_counter";
-  {
-    // Test priority_regex.
-    MetricPrometheusOptions opts;
-    opts.priority_regex_string = "t_c.*r";
-    std::stringstream output;
-    PrometheusWriter writer(&output, opts);
-    ASSERT_OK(registry_.WriteForPrometheus(&writer, opts));
-    MetricAggregationMap* filter =
-        writer.TEST_GetPrometheusMetricFilter()->TEST_GetAggregationMap();
-    ASSERT_EQ(kServerLevel | kStreamLevel, (*filter)[kGaugeName]);
-    ASSERT_EQ(kTableLevel | kServerLevel | kStreamLevel, (*filter)[kCounterName]);
-    int line_num = 0;
-    std::string line;
-    while (std::getline(output, line)) {
-      if (line.find("table_id=") != std::string::npos) {
-        // If line contain table id, it means metric is on table level.
-        ASSERT_TRUE(line.starts_with(kCounterName));
-      } else {
-        // Otherwise, it is server level.
-        ASSERT_TRUE(line.starts_with(kGaugeName));
-      }
-      line_num++;
-    }
-    ASSERT_EQ(2, line_num);
-  }
-  {
-    // Test default.
-    MetricPrometheusOptions opts;
-    std::stringstream output;
-    PrometheusWriter writer(&output, opts);
-    ASSERT_OK(registry_.WriteForPrometheus(&writer, opts));
-    MetricAggregationMap* filter =
-        writer.TEST_GetPrometheusMetricFilter()->TEST_GetAggregationMap();
-    EXPECT_EQ(kTableLevel | kServerLevel | kStreamLevel, (*filter)[kGaugeName]);
-    EXPECT_EQ(kTableLevel | kServerLevel | kStreamLevel, (*filter)[kCounterName]);
-    int line_num = 0;
-    std::string line;
-    while (std::getline(output, line)) {
-      ASSERT_TRUE(line.find("table_id=") != std::string::npos);
-      line_num++;
-    }
-    ASSERT_EQ(2, line_num);
-  }
-}
 } // namespace yb

src/yb/util/metrics_writer.cc

@@ -25,8 +25,7 @@ PrometheusWriter::PrometheusWriter(std::stringstream* output,
       timestamp_(std::chrono::duration_cast<std::chrono::milliseconds>(
           std::chrono::system_clock::now().time_since_epoch()).count()),
       export_help_and_type_(opts.export_help_and_type),
-      prometheus_metric_filter_(
-          std::make_unique<PrometheusMetricFilter>(opts.priority_regex_string)) {}
+      prometheus_metric_filter_(CreatePrometheusMetricFilter(opts)) {}
 
 PrometheusWriter::~PrometheusWriter() {}
 
@@ -108,10 +107,11 @@ Status PrometheusWriter::WriteSingleEntry(
     const std::string& name,
     int64_t value,
     AggregationFunction aggregation_function,
-    AggregationLevels aggregation_levels,
+    AggregationLevels default_levels,
     const char* type,
     const char* description) {
-  aggregation_levels &= prometheus_metric_filter_->GetAggregationLevels(name);
+  AggregationLevels aggregation_levels =
+      prometheus_metric_filter_->GetAggregationLevels(name, default_levels);
 
   auto metric_type_it = attr.find("metric_type");
   DCHECK(metric_type_it != attr.end());
@@ -140,8 +140,6 @@ Status PrometheusWriter::WriteSingleEntry(
       AddAggregatedEntry(tablet_id_it->second, attr, name, value, aggregation_function,
           type, description);
     }
-    // Currently, if a metric is exported on table level, it shouldn't be on other levels.
-    return Status::OK();
   }
 
   if (aggregation_levels & kServerLevel) {

src/yb/util/metrics_writer.h

@@ -56,7 +56,7 @@ class PrometheusWriter {
       const std::string& name,
       int64_t value,
       AggregationFunction aggregation_function,
-      AggregationLevels aggregation_levels,
+      AggregationLevels default_levels,
       const char* type = "unknown",
       const char* description = "unknown");