SOLR-17806: Migrate ADMIN node registry metrics to OTEL #3444

mlbiscoc · 2025-07-28T19:58:25Z

https://issues.apache.org/jira/browse/SOLR-17806

Migrate node registry ADMIN metrics and remove Dropwizard initializations from RequestHandlerBase.

This does not include aggregateNodeLevelMetricsEnabled

# HELP solr_node_requests_total Http Solr node requests
# TYPE solr_node_requests_total counter
solr_node_requests_total{category="ADMIN",handler="/admin/authorization",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_total{category="ADMIN",handler="/admin/collections",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_total{category="ADMIN",handler="/admin/configs",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_total{category="ADMIN",handler="/admin/cores",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_total{category="ADMIN",handler="/admin/info",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_total{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr"} 1.0
solr_node_requests_total{category="ADMIN",handler="/admin/zookeeper",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_total{category="ADMIN",handler="/admin/zookeeper/status",otel_scope_name="org.apache.solr"} 0.0
# HELP solr_node_requests_errors_total HTTP Solr node request errors
# TYPE solr_node_requests_errors_total counter
solr_node_requests_errors_total{category="ADMIN",handler="/admin/authorization",otel_scope_name="org.apache.solr",source="client"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/authorization",otel_scope_name="org.apache.solr",source="server"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/collections",otel_scope_name="org.apache.solr",source="client"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/collections",otel_scope_name="org.apache.solr",source="server"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/configs",otel_scope_name="org.apache.solr",source="client"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/configs",otel_scope_name="org.apache.solr",source="server"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/cores",otel_scope_name="org.apache.solr",source="client"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/cores",otel_scope_name="org.apache.solr",source="server"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/info",otel_scope_name="org.apache.solr",source="client"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/info",otel_scope_name="org.apache.solr",source="server"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",source="client"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",source="server"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/zookeeper",otel_scope_name="org.apache.solr",source="client"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/zookeeper",otel_scope_name="org.apache.solr",source="server"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/zookeeper/status",otel_scope_name="org.apache.solr",source="client"} 0.0
solr_node_requests_errors_total{category="ADMIN",handler="/admin/zookeeper/status",otel_scope_name="org.apache.solr",source="server"} 0.0
# HELP solr_node_requests_timeout_total HTTP Solr node request timeouts
# TYPE solr_node_requests_timeout_total counter
solr_node_requests_timeout_total{category="ADMIN",handler="/admin/authorization",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_timeout_total{category="ADMIN",handler="/admin/collections",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_timeout_total{category="ADMIN",handler="/admin/configs",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_timeout_total{category="ADMIN",handler="/admin/cores",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_timeout_total{category="ADMIN",handler="/admin/info",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_timeout_total{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_timeout_total{category="ADMIN",handler="/admin/zookeeper",otel_scope_name="org.apache.solr"} 0.0
solr_node_requests_timeout_total{category="ADMIN",handler="/admin/zookeeper/status",otel_scope_name="org.apache.solr"} 0.0
# HELP solr_node_requests_times_milliseconds HTTP Solr node request times
# TYPE solr_node_requests_times_milliseconds histogram
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="0.0"} 0
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="5.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="10.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="25.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="50.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="75.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="100.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="250.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="500.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="750.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="1000.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="2500.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="5000.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="7500.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="10000.0"} 1
solr_node_requests_times_milliseconds_bucket{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr",le="+Inf"} 1
solr_node_requests_times_milliseconds_count{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr"} 1
solr_node_requests_times_milliseconds_sum{category="ADMIN",handler="/admin/metrics",otel_scope_name="org.apache.solr"} 4.0

dsmiley

Does Prometheus expose these counts as floating point instead as integer (long)?

dsmiley · 2025-07-29T01:46:10Z

solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java

nice to finally see just otel and not also dropwizard simultaneously

dsmiley · 2025-07-29T01:50:19Z

solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java

      }
    }
  }
+
+  private Double getSelectRequestCount(SolrCore core) {


why a Double instead of long ?

solr/solrj/src/test/org/apache/solr/client/solrj/SolrJMetricTestUtils.java

dsmiley · 2025-07-29T01:55:15Z

solr/solrj/src/test/org/apache/solr/client/solrj/SolrJMetricTestUtils.java

+              SolrRequest.METHOD.GET,
+              "/admin/metrics",
+              SolrRequest.SolrRequestType.ADMIN,
+              new ModifiableSolrParams().set("wt", "prometheus"));


FYI try SolrParams.of("wt", "prometheus")

dsmiley · 2025-07-29T01:57:26Z

solr/solrj/src/test/org/apache/solr/client/solrj/SolrJMetricTestUtils.java

+
+        // Sum both client and server errors
+        return line.stream()
+            .map(s -> Double.parseDouble(s.substring(s.lastIndexOf(" ") + 1).trim()))


see Stream.mapToDouble and Stream.mapToLong

dsmiley · 2025-07-29T01:58:33Z

solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientRetryTest.java

-      System.out.println(namedList);
-      NamedList<?> metrics = (NamedList<?>) namedList.get("metrics");
-      assertEquals(1L, metrics.get(updateRequestCountKey));
+      assertEquals(1.0, getPrometheusMetricValue(solrClient, prometheusMetric), 0.0);


nice utility method

dsmiley · 2025-07-29T22:41:02Z

solr/solrj/src/test/org/apache/solr/client/solrj/SolrJMetricTestUtils.java

+      return output
+          .lines()
+          .filter(l -> l.startsWith(metricName))
+          .mapToDouble(s -> Double.parseDouble(s.substring(s.lastIndexOf(" ") + 1).trim()))


The plus one results in a space that you then remove with a trim. So can't you skip both and simplify?

mlbiscoc added 5 commits July 28, 2025 14:49

Migrate admin node metrics to OTEL

308d617

Cleanup

73863f2

Fix up some tests

5d6ffe8

Move some things around

00aac9f

Bad apple test mbeans handler

7062893

github-actions bot added client:solrj tests cat:search cat:cloud cat:api cat:metrics labels Jul 28, 2025

dsmiley reviewed Jul 29, 2025

View reviewed changes

mlbiscoc added 2 commits July 29, 2025 15:55

mapToDouble

6b3a794

Merge branch 'feature/SOLR-17458' into admin-node-metrics

3728f3d

dsmiley reviewed Jul 29, 2025

View reviewed changes

Skip whitespace trimming and skipping

f24c328

dsmiley approved these changes Jul 31, 2025

View reviewed changes

mlbiscoc merged commit 1a4daae into apache:feature/SOLR-17458 Aug 6, 2025
1 check passed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

SOLR-17806: Migrate ADMIN node registry metrics to OTEL #3444

SOLR-17806: Migrate ADMIN node registry metrics to OTEL #3444

mlbiscoc commented Jul 28, 2025

Uh oh!

dsmiley left a comment

Uh oh!

dsmiley Jul 29, 2025

Uh oh!

dsmiley Jul 29, 2025

Uh oh!

Uh oh!

dsmiley Jul 29, 2025

Uh oh!

dsmiley Jul 29, 2025

Uh oh!

dsmiley Jul 29, 2025

Uh oh!

dsmiley Jul 29, 2025

Uh oh!

Uh oh!

Uh oh!

SOLR-17806: Migrate ADMIN node registry metrics to OTEL #3444

SOLR-17806: Migrate ADMIN node registry metrics to OTEL #3444

Conversation

mlbiscoc commented Jul 28, 2025

Uh oh!

dsmiley left a comment

Choose a reason for hiding this comment

Uh oh!

dsmiley Jul 29, 2025

Choose a reason for hiding this comment

Uh oh!

dsmiley Jul 29, 2025

Choose a reason for hiding this comment

Uh oh!

Uh oh!

dsmiley Jul 29, 2025

Choose a reason for hiding this comment

Uh oh!

dsmiley Jul 29, 2025

Choose a reason for hiding this comment

Uh oh!

dsmiley Jul 29, 2025

Choose a reason for hiding this comment

Uh oh!

dsmiley Jul 29, 2025

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!