HBASE-25853 Backport HBASE-22120 "Replace HTrace with OpenTelemetry" to branch-2

bin/hbase

@@ -487,6 +487,11 @@ add_jdk11_deps_to_classpath() {
   done
 }
 
+enable_trace() {
+  agent_jar=$(find lib/trace -type f -name "opentelemetry-javaagent-*")
+  HBASE_OPTS="$HBASE_OPTS -javaagent:$agent_jar $HBASE_TRACE_OPTS"
+}
+
 #Add the development env class path stuff
 if $in_dev_env; then
   add_maven_deps_to_classpath "cached_classpath.txt"
@@ -764,6 +769,11 @@ elif [ "${DEBUG}" = "true" ]; then
   echo "JDK11 jars skipped from classpath."
 fi
 
+if [[ -n "${HBASE_TRACE_OPTS}" ]]; then
+  echo "Attach opentelemetry agent to enable trace"
+  enable_trace
+fi
+
 # Have JVM dump heap if we run out of memory.  Files will be 'launch directory'
 # and are named like the following: java_pid21612.hprof. Apparently it doesn't
 # 'cost' to have this flag enabled. Its a 1.6 flag only. See:

conf/hbase-env.cmd

@@ -81,3 +81,8 @@ set HBASE_OPTS=%HBASE_OPTS% "-XX:+UseConcMarkSweepGC" "-Djava.net.preferIPv4Stac
 
 @rem Tell HBase whether it should manage it's own instance of ZooKeeper or not.
 @rem set HBASE_MANAGES_ZK=true
+
+@rem Uncomment to enable trace, you can change the options to use other exporters such as jaeger or
+@rem zipkin. See https://github.com/open-telemetry/opentelemetry-java-instrumentation on how to
+@rem configure exporters and other components through system properties.
+@rem set HBASE_TRACE_OPTS="-Dotel.resource.attributes=service.name=HBase -Dotel.traces.exporter=logging otel.metrics.exporter=none"

conf/hbase-env.sh

@@ -144,3 +144,8 @@
 
 # Additional argments passed to jshell invocation
 # export HBASE_JSHELL_ARGS="--startup DEFAULT --startup PRINTING --startup hbase_startup.jsh"
+
+# Uncomment to enable trace, you can change the options to use other exporters such as jaeger or
+# zipkin. See https://github.com/open-telemetry/opentelemetry-java-instrumentation on how to
+# configure exporters and other components through system properties.
+# export HBASE_TRACE_OPTS="-Dotel.resource.attributes=service.name=HBase -Dotel.traces.exporter=logging otel.metrics.exporter=none"

hbase-assembly/pom.xml

@@ -339,6 +339,12 @@
       <groupId>log4j</groupId>
       <artifactId>log4j</artifactId>
     </dependency>
+    <!-- Include OpenTelemetry agent -->
+    <dependency>
+      <groupId>io.opentelemetry.javaagent</groupId>
+      <artifactId>opentelemetry-javaagent</artifactId>
+      <classifier>all</classifier>
+    </dependency>
   </dependencies>
   <profiles>
     <profile>

hbase-assembly/src/main/assembly/client.xml

@@ -65,6 +65,7 @@
               <exclude>org.slf4j:jcl-over-slf4j</exclude>
               <exclude>org.slf4j:jul-to-slf4j</exclude>
               <exclude>org.slf4j:slf4j-log4j12</exclude>
+              <exclude>io.opentelemetry.javaagent:*</exclude>
             </excludes>
           </dependencySet>
         </dependencySets>
@@ -153,6 +154,13 @@
         <include>org.slf4j:jcl-over-slf4j</include>
         <include>org.slf4j:jul-to-slf4j</include>
         <include>org.slf4j:slf4j-log4j12</include>
+        <include>io.opentelemetry:*</include>
+      </includes>
+    </dependencySet>
+    <dependencySet>
+      <outputDirectory>lib/trace</outputDirectory>
+      <includes>
+        <include>io.opentelemetry.javaagent:*</include>
       </includes>
     </dependencySet>
   </dependencySets>

hbase-assembly/src/main/assembly/hadoop-three-compat.xml

@@ -112,6 +112,7 @@
               <exclude>org.apache.yetus:audience-annotations</exclude>
               <exclude>org.slf4j:slf4j-api</exclude>
               <exclude>org.slf4j:slf4j-log4j12</exclude>
+              <exclude>io.opentelemetry.javaagent:*</exclude>
             </excludes>
           </dependencySet>
         </dependencySets>
@@ -212,6 +213,7 @@
         <include>org.slf4j:jcl-over-slf4j</include>
         <include>org.slf4j:jul-to-slf4j</include>
         <include>org.slf4j:slf4j-log4j12</include>
+        <include>io.opentelemetry:*</include>
       </includes>
     </dependencySet>
     <dependencySet>
@@ -256,6 +258,12 @@
         <include>jakarta.jws:jakarta.jws-api</include>
       </includes>
     </dependencySet>
+    <dependencySet>
+      <outputDirectory>lib/trace</outputDirectory>
+      <includes>
+        <include>io.opentelemetry.javaagent:*</include>
+      </includes>
+    </dependencySet>
   </dependencySets>
 
 </assembly>

hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSTestBase.java

@@ -19,11 +19,9 @@
 
 import java.io.File;
 import java.io.IOException;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
-import org.apache.hadoop.hbase.trace.TraceUtil;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -104,7 +102,6 @@ protected static void startMiniDFSCluster(int servers) throws IOException {
     org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.impl.MetricsSystemImpl.class)
       .setLevel(org.apache.log4j.Level.ERROR);
 
-    TraceUtil.initTracer(conf);
     CLUSTER = new MiniDFSCluster.Builder(conf).numDataNodes(servers).build();
     CLUSTER.waitClusterUp();
   }

hbase-client/pom.xml

@@ -145,8 +145,8 @@
       <artifactId>zookeeper</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.apache.htrace</groupId>
-      <artifactId>htrace-core4</artifactId>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-api</artifactId>
     </dependency>
     <dependency>
       <groupId>org.jruby.jcodings</groupId>
@@ -156,6 +156,16 @@
       <groupId>org.jruby.joni</groupId>
       <artifactId>joni</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-testing</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>jcl-over-slf4j</artifactId>

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java

@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import static org.apache.hadoop.hbase.trace.TraceUtil.trace;
+import static org.apache.hadoop.hbase.trace.TraceUtil.tracedFuture;
 import static org.apache.hadoop.hbase.util.FutureUtils.addListener;
 
 import com.google.errorprone.annotations.RestrictedApi;
@@ -237,38 +239,48 @@ private static RegionLocations transformMetaRegionLocations(GetMetaRegionLocatio
 
   @Override
   public CompletableFuture<RegionLocations> getMetaRegionLocations() {
-    return this
-      .<GetMetaRegionLocationsResponse> call((c, s, d) -> s.getMetaRegionLocations(c,
-        GetMetaRegionLocationsRequest.getDefaultInstance(), d), r -> r.getMetaLocationsCount() != 0,
+    return tracedFuture(
+      () -> this
+        .<GetMetaRegionLocationsResponse> call(
+          (c, s, d) -> s.getMetaRegionLocations(c,
+            GetMetaRegionLocationsRequest.getDefaultInstance(), d),
+          r -> r.getMetaLocationsCount() != 0,
         "getMetaLocationsCount")
-      .thenApply(AbstractRpcBasedConnectionRegistry::transformMetaRegionLocations);
+        .thenApply(AbstractRpcBasedConnectionRegistry::transformMetaRegionLocations),
+      getClass().getSimpleName() + ".getMetaRegionLocations");
   }
 
   @Override
   public CompletableFuture<String> getClusterId() {
-    return this
-      .<GetClusterIdResponse> call(
-        (c, s, d) -> s.getClusterId(c, GetClusterIdRequest.getDefaultInstance(), d),
-        GetClusterIdResponse::hasClusterId, "getClusterId()")
-      .thenApply(GetClusterIdResponse::getClusterId);
+    return tracedFuture(
+      () -> this
+        .<GetClusterIdResponse> call(
+          (c, s, d) -> s.getClusterId(c, GetClusterIdRequest.getDefaultInstance(), d),
+          GetClusterIdResponse::hasClusterId, "getClusterId()")
+        .thenApply(GetClusterIdResponse::getClusterId),
+      getClass().getSimpleName() + ".getClusterId");
   }
 
   @Override
   public CompletableFuture<ServerName> getActiveMaster() {
-    return this
-      .<GetActiveMasterResponse> call(
-        (c, s, d) -> s.getActiveMaster(c, GetActiveMasterRequest.getDefaultInstance(), d),
-        GetActiveMasterResponse::hasServerName, "getActiveMaster()")
-      .thenApply(resp -> ProtobufUtil.toServerName(resp.getServerName()));
+    return tracedFuture(
+      () -> this
+        .<GetActiveMasterResponse>call(
+          (c, s, d) -> s.getActiveMaster(c, GetActiveMasterRequest.getDefaultInstance(), d),
+          GetActiveMasterResponse::hasServerName, "getActiveMaster()")
+        .thenApply(resp -> ProtobufUtil.toServerName(resp.getServerName())),
+      getClass().getSimpleName() + ".getActiveMaster");
   }
 
   @Override
   public void close() {
-    if (registryEndpointRefresher != null) {
-      registryEndpointRefresher.stop();
-    }
-    if (rpcClient != null) {
-      rpcClient.close();
-    }
+    trace(() -> {
+      if (registryEndpointRefresher != null) {
+        registryEndpointRefresher.stop();
+      }
+      if (rpcClient != null) {
+        rpcClient.close();
+      }
+    }, getClass().getSimpleName() + ".close");
   }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnection.java

@@ -64,8 +64,8 @@ public interface AsyncConnection extends Closeable {
   /**
    * Retrieve an {@link AsyncTable} implementation for accessing a table.
    * <p>
-   * The returned instance will use default configs. Use {@link #getTableBuilder(TableName)} if
-   * you want to customize some configs.
+   * The returned instance will use default configs. Use {@link #getTableBuilder(TableName)} if you
+   * want to customize some configs.
    * <p>
    * This method no longer checks table existence. An exception will be thrown if the table does not
    * exist only when the first operation is attempted.