Add OTLP metrics support (#1492)

* Add OTLP metrics support

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

* Don't automatically push to grpc. Only allow if pushEnabled is set to true

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

* Code review

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

* Missed refactoring

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

* Add missing header

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

*  Don't make otel depend on the push enabled flag

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

* Expose JUL logs to log4j2

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

* Code review fixes and make sure not to start push gateway if not set to prometheus

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

* spotless

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

* Fix unit test

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

* code review feedback

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

* Add changelog entry

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

CHANGELOG.md

@@ -12,6 +12,7 @@
 * Updated the libraries for secp256k1 and AltBN series precompiles. These updates provide significant performance improvements to those areas. [\#1499](https://github.com/hyperledger/besu/pull/1499)
 * Provide MegaGas/second measurements in the log when doing a full block import, such as the catch up phase of a fast sync. [\#1512](https://github.com/hyperledger/besu/pull/1512)
 * Added new endpoints to get miner data, `eth_getMinerDataByBlockHash` and `eth_getMinerDataByBlockNumber`. [\#1538](https://github.com/hyperledger/besu/pull/1538)
+* Added direct support for OpenTelemetry metrics [\#1492](https://github.com/hyperledger/besu/pull/1492)
 
 ### Bug Fixes
 

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -34,8 +34,8 @@
 import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
 import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider;
 import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder;
+import org.hyperledger.besu.metrics.MetricsSystemFactory;
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
-import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem;
 import org.hyperledger.besu.plugin.services.BesuConfiguration;
 import org.hyperledger.besu.plugin.services.BesuEvents;
 import org.hyperledger.besu.plugin.services.PicoCLIOptions;
@@ -128,7 +128,7 @@ public void startNode(final BesuNode node) {
                     node, storageService, securityModuleService, commonPluginConfiguration));
 
     final ObservableMetricsSystem metricsSystem =
-        PrometheusMetricsSystem.init(node.getMetricsConfiguration());
+        MetricsSystemFactory.create(node.getMetricsConfiguration());
     final List<EnodeURL> bootnodes =
         node.getConfiguration().getBootnodes().stream()
             .map(EnodeURL::fromURI)

besu/build.gradle

@@ -70,6 +70,7 @@ dependencies {
 
 
   runtimeOnly 'org.apache.logging.log4j:log4j-slf4j-impl'
+  runtimeOnly 'org.apache.logging.log4j:log4j-jul'
   runtimeOnly 'com.splunk.logging:splunk-library-javalogging'
   runtimeOnly 'org.fusesource.jansi:jansi' // for color logging in windows
 

besu/src/main/java/org/hyperledger/besu/Runner.java

@@ -25,7 +25,7 @@
 import org.hyperledger.besu.ethereum.p2p.peers.EnodeURL;
 import org.hyperledger.besu.ethereum.stratum.StratumServer;
 import org.hyperledger.besu.ethstats.EthStatsService;
-import org.hyperledger.besu.metrics.prometheus.MetricsService;
+import org.hyperledger.besu.metrics.MetricsService;
 import org.hyperledger.besu.nat.NatService;
 
 import java.io.File;

besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java

@@ -93,9 +93,9 @@
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 import org.hyperledger.besu.ethstats.EthStatsService;
 import org.hyperledger.besu.ethstats.util.NetstatsUrl;
+import org.hyperledger.besu.metrics.MetricsService;
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
-import org.hyperledger.besu.metrics.prometheus.MetricsService;
 import org.hyperledger.besu.nat.NatMethod;
 import org.hyperledger.besu.nat.NatService;
 import org.hyperledger.besu.nat.core.NatManager;
@@ -603,10 +603,7 @@ public Runner build() {
       createPrivateTransactionObserver(subscriptionManager, privacyParameters);
     }
 
-    Optional<MetricsService> metricsService = Optional.empty();
-    if (metricsConfiguration.isEnabled() || metricsConfiguration.isPushEnabled()) {
-      metricsService = Optional.of(createMetricsService(vertx, metricsConfiguration));
-    }
+    Optional<MetricsService> metricsService = createMetricsService(vertx, metricsConfiguration);
 
     final Optional<EthStatsService> ethStatsService;
     if (!Strings.isNullOrEmpty(ethstatsUrl)) {
@@ -884,7 +881,7 @@ private WebSocketService createWebsocketService(
     return new WebSocketService(vertx, configuration, websocketRequestHandler);
   }
 
-  private MetricsService createMetricsService(
+  private Optional<MetricsService> createMetricsService(
       final Vertx vertx, final MetricsConfiguration configuration) {
     return MetricsService.create(vertx, configuration, metricsSystem);
   }

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -28,6 +28,7 @@
 import static org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration.DEFAULT_WEBSOCKET_PORT;
 import static org.hyperledger.besu.ethereum.permissioning.QuorumPermissioningConfiguration.QIP714_DEFAULT_BLOCK;
 import static org.hyperledger.besu.metrics.BesuMetricCategory.DEFAULT_METRIC_CATEGORIES;
+import static org.hyperledger.besu.metrics.MetricsProtocol.PROMETHEUS;
 import static org.hyperledger.besu.metrics.prometheus.MetricsConfiguration.DEFAULT_METRICS_PORT;
 import static org.hyperledger.besu.metrics.prometheus.MetricsConfiguration.DEFAULT_METRICS_PUSH_PORT;
 import static org.hyperledger.besu.nat.kubernetes.KubernetesNatManager.DEFAULT_BESU_SERVICE_NAME_FILTER;
@@ -116,10 +117,11 @@
 import org.hyperledger.besu.ethereum.worldstate.PrunerConfiguration;
 import org.hyperledger.besu.metrics.BesuMetricCategory;
 import org.hyperledger.besu.metrics.MetricCategoryRegistryImpl;
+import org.hyperledger.besu.metrics.MetricsProtocol;
+import org.hyperledger.besu.metrics.MetricsSystemFactory;
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.metrics.StandardMetricCategory;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
-import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem;
 import org.hyperledger.besu.metrics.vertx.VertxMetricsAdapterFactory;
 import org.hyperledger.besu.nat.NatMethod;
 import org.hyperledger.besu.plugin.services.BesuConfiguration;
@@ -648,6 +650,13 @@ void setBannedNodeIds(final List<String> values) {
       description = "Set to start the metrics exporter (default: ${DEFAULT-VALUE})")
   private final Boolean isMetricsEnabled = false;
 
+  @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings.
+  @Option(
+      names = {"--metrics-protocol"},
+      description =
+          "Metrics protocol, one of PROMETHEUS, OPENTELEMETRY or NONE. (default: ${DEFAULT-VALUE})")
+  private MetricsProtocol metricsProtocol = PROMETHEUS;
+
   @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings.
   @Option(
       names = {"--metrics-host"},
@@ -1022,7 +1031,7 @@ void setBannedNodeIds(final List<String> values) {
   private BesuController besuController;
   private BesuConfiguration pluginCommonConfiguration;
   private final Supplier<ObservableMetricsSystem> metricsSystem =
-      Suppliers.memoize(() -> PrometheusMetricsSystem.init(metricsConfiguration()));
+      Suppliers.memoize(() -> MetricsSystemFactory.create(metricsConfiguration()));
   private Vertx vertx;
   private EnodeDnsConfiguration enodeDnsConfiguration;
 
@@ -1147,6 +1156,7 @@ private void registerConverters() {
     commandLine.registerConverter(Bytes.class, Bytes::fromHexString);
     commandLine.registerConverter(Level.class, Level::valueOf);
     commandLine.registerConverter(SyncMode.class, SyncMode::fromString);
+    commandLine.registerConverter(MetricsProtocol.class, MetricsProtocol::fromString);
     commandLine.registerConverter(UInt256.class, (arg) -> UInt256.valueOf(new BigInteger(arg)));
     commandLine.registerConverter(Wei.class, (arg) -> Wei.of(Long.parseUnsignedLong(arg)));
     commandLine.registerConverter(PositiveNumber.class, PositiveNumber::fromString);
@@ -1734,6 +1744,7 @@ public MetricsConfiguration metricsConfiguration() {
         .enabled(isMetricsEnabled)
         .host(metricsHost)
         .port(metricsPort)
+        .protocol(metricsProtocol)
         .metricCategories(metricCategories)
         .pushEnabled(isMetricsPushEnabled)
         .pushHost(metricsPushHost)

besu/src/main/java/org/hyperledger/besu/cli/subcommands/blocks/BlocksSubCommand.java

@@ -33,8 +33,8 @@
 import org.hyperledger.besu.ethereum.core.Address;
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.core.Wei;
+import org.hyperledger.besu.metrics.MetricsService;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
-import org.hyperledger.besu.metrics.prometheus.MetricsService;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -434,18 +434,13 @@ private Optional<Long> getEndBlock() {
   }
 
   private static Optional<MetricsService> initMetrics(final BlocksSubCommand parentCommand) {
-    Optional<MetricsService> metricsService = Optional.empty();
     final MetricsConfiguration metricsConfiguration =
         parentCommand.parentCommand.metricsConfiguration();
-    if (metricsConfiguration.isEnabled() || metricsConfiguration.isPushEnabled()) {
-      metricsService =
-          Optional.of(
-              MetricsService.create(
-                  Vertx.vertx(),
-                  metricsConfiguration,
-                  parentCommand.parentCommand.getMetricsSystem()));
-      metricsService.ifPresent(MetricsService::start);
-    }
+
+    Optional<MetricsService> metricsService =
+        MetricsService.create(
+            Vertx.vertx(), metricsConfiguration, parentCommand.parentCommand.getMetricsSystem());
+    metricsService.ifPresent(MetricsService::start);
     return metricsService;
   }
 }

besu/src/test/resources/everything_config.toml

@@ -92,6 +92,7 @@ rpc-ws-authentication-jwt-public-key-file="none"
 
 # Prometheus Metrics Endpoint
 metrics-enabled=false
+metrics-protocol="prometheus"
 metrics-host="8.6.7.5"
 metrics-port=309
 metrics-category=["RPC"]
@@ -164,4 +165,4 @@ Xethstats-contact="contact@mail.n"
 
 # feature flags
 Xsecp256k1-native-enabled=false
-Xaltbn128-native-enabled=false
+Xaltbn128-native-enabled=false

build.gradle

@@ -469,6 +469,8 @@ applicationDefaultJvmArgs = [
   // We shutdown log4j ourselves, as otherwise this shutdown hook runs before our own and whatever
   // happens during shutdown is not logged.
   '-Dlog4j.shutdownHookEnabled=false',
+  // Redirect java.util.logging loggers to use log4j2.
+  '-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager',
   // Suppress Java JPMS warnings.  Document the reason for each suppression.
   // Bouncy Castle needs access to sun.security.provider, which is not open by default.
   '--add-opens',

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java

@@ -28,9 +28,9 @@
 import org.hyperledger.besu.ethereum.core.TransactionReceipt;
 import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage;
+import org.hyperledger.besu.metrics.MetricsSystemFactory;
 import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
-import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;
 
@@ -164,7 +164,7 @@ public void initializeReadOnly_withGiantDifficultyAndLiveMetrics() {
     final Blockchain blockchain =
         DefaultBlockchain.create(
             createStorage(kvStore),
-            PrometheusMetricsSystem.init(MetricsConfiguration.builder().enabled(true).build()),
+            MetricsSystemFactory.create(MetricsConfiguration.builder().enabled(true).build()),
             0);
 
     for (int i = 0; i < blocks.size(); i++) {

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java

@@ -185,7 +185,7 @@ public void run() {
                           : GenesisFileModule.createGenesisModule(genesisFile)
                       : GenesisFileModule.createGenesisModule(network))
               .evmToolCommandOptionsModule(daggerOptions)
-              .metricsSystemModule(new PrometheusMetricsSystemModule())
+              .metricsSystemModule(new MetricsSystemModule())
               .build();
 
       final BlockHeader blockHeader =

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/MetricsSystemModule.java

@@ -16,6 +16,8 @@
 
 package org.hyperledger.besu.evmtool;
 
+import org.hyperledger.besu.metrics.MetricsSystemFactory;
+import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 
 import dagger.Module;
@@ -27,6 +29,6 @@ public class MetricsSystemModule {
 
   @Provides
   MetricsSystem getMetricsSystem() {
-    throw new RuntimeException("Abstract");
+    return MetricsSystemFactory.create(MetricsConfiguration.builder().build());
   }
 }

gradle/versions.gradle

@@ -45,12 +45,19 @@ dependencyManagement {
 
     dependency 'info.picocli:picocli:4.5.0'
 
+    dependency 'io.grpc:grpc-netty:1.33.0'
+
     dependency 'io.kubernetes:client-java:5.0.0'
 
     dependency 'io.pkts:pkts-core:3.0.7'
 
     dependency group: 'io.opentelemetry.instrumentation.auto', name: 'opentelemetry-javaagent', version: '0.8.0', classifier: 'all'
 
+    dependency 'io.opentelemetry:opentelemetry-api:0.9.1'
+    dependency 'io.opentelemetry:opentelemetry-sdk:0.9.1'
+    dependency 'io.opentelemetry:opentelemetry-exporters-otlp:0.9.1'
+    dependency 'io.opentelemetry:opentelemetry-extension-runtime-metrics:0.9.1'
+
     dependency 'io.prometheus:simpleclient:0.9.0'
     dependency 'io.prometheus:simpleclient_common:0.9.0'
     dependency 'io.prometheus:simpleclient_hotspot:0.9.0'
@@ -75,6 +82,7 @@ dependencyManagement {
 
     dependency 'org.apache.logging.log4j:log4j-api:2.13.3'
     dependency 'org.apache.logging.log4j:log4j-core:2.13.3'
+    dependency 'org.apache.logging.log4j:log4j-jul:2.13.3'
     dependency 'org.apache.logging.log4j:log4j-slf4j-impl:2.13.3'
     dependency 'org.fusesource.jansi:jansi:1.8'
 

metrics/core/build.gradle

@@ -38,6 +38,12 @@ dependencies {
   implementation project(':plugin-api')
 
   implementation 'com.google.guava:guava'
+  implementation 'io.grpc:grpc-netty'
+  implementation 'io.opentelemetry:opentelemetry-api'
+  implementation 'io.opentelemetry:opentelemetry-sdk'
+  implementation 'io.opentelemetry:opentelemetry-extension-runtime-metrics'
+  implementation 'io.opentelemetry:opentelemetry-exporters-otlp'
+
   implementation 'io.prometheus:simpleclient'
   implementation 'io.prometheus:simpleclient_common'
   implementation 'io.prometheus:simpleclient_hotspot'

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/PrometheusMetricsSystemModule.java → metrics/core/src/main/java/org/hyperledger/besu/metrics/MetricsProtocol.java

@@ -11,19 +11,21 @@
  * specific language governing permissions and limitations under the License.
  *
  * SPDX-License-Identifier: Apache-2.0
- *
  */
+package org.hyperledger.besu.metrics;
 
-package org.hyperledger.besu.evmtool;
-
-import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
-import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem;
-import org.hyperledger.besu.plugin.services.MetricsSystem;
-
-public class PrometheusMetricsSystemModule extends MetricsSystemModule {
+/** Enumeration of metrics protocols supported by Besu. */
+public enum MetricsProtocol {
+  PROMETHEUS,
+  OPENTELEMETRY,
+  NONE;
 
-  @Override
-  public MetricsSystem getMetricsSystem() {
-    return PrometheusMetricsSystem.init(MetricsConfiguration.builder().build());
+  public static MetricsProtocol fromString(final String str) {
+    for (final MetricsProtocol mode : MetricsProtocol.values()) {
+      if (mode.name().equalsIgnoreCase(str)) {
+        return mode;
+      }
+    }
+    return null;
   }
 }

metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/MetricsService.java → metrics/core/src/main/java/org/hyperledger/besu/metrics/MetricsService.java

@@ -12,27 +12,47 @@
  *
  * SPDX-License-Identifier: Apache-2.0
  */
-package org.hyperledger.besu.metrics.prometheus;
+package org.hyperledger.besu.metrics;
 
+import org.hyperledger.besu.metrics.opentelemetry.MetricsOtelGrpcPushService;
+import org.hyperledger.besu.metrics.opentelemetry.OpenTelemetrySystem;
+import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
+import org.hyperledger.besu.metrics.prometheus.MetricsHttpService;
+import org.hyperledger.besu.metrics.prometheus.MetricsPushGatewayService;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 
 import io.vertx.core.Vertx;
 
+/**
+ * Service responsible for exposing metrics to the outside, either through a port and network
+ * interface or pushing.
+ */
 public interface MetricsService {
 
-  static MetricsService create(
+  static Optional<MetricsService> create(
       final Vertx vertx,
       final MetricsConfiguration configuration,
       final MetricsSystem metricsSystem) {
-    if (configuration.isEnabled()) {
-      return new MetricsHttpService(vertx, configuration, metricsSystem);
-    } else if (configuration.isPushEnabled()) {
-      return new MetricsPushGatewayService(configuration, metricsSystem);
+    if (configuration.getProtocol() == MetricsProtocol.PROMETHEUS) {
+      if (configuration.isEnabled()) {
+        return Optional.of(new MetricsHttpService(vertx, configuration, metricsSystem));
+      } else if (configuration.isPushEnabled()) {
+        return Optional.of(new MetricsPushGatewayService(configuration, metricsSystem));
+      } else {
+        return Optional.empty();
+      }
+    } else if (configuration.getProtocol() == MetricsProtocol.OPENTELEMETRY) {
+      if (configuration.isEnabled()) {
+        return Optional.of(
+            new MetricsOtelGrpcPushService(configuration, (OpenTelemetrySystem) metricsSystem));
+      } else {
+        return Optional.empty();
+      }
     } else {
-      throw new RuntimeException("No metrics service enabled.");
+      return Optional.empty();
     }
   }