[Abandoned] HADOOP-16612 Track Azure Blob File System client-perceived latency

BUILDING.txt

@@ -6,7 +6,6 @@ Requirements:
 * Unix System
 * JDK 1.8
 * Maven 3.3 or later
-* ProtocolBuffer 3.7.1
 * CMake 3.1 or newer (if compiling native code)
 * Zlib devel (if compiling native code)
 * Cyrus SASL devel (if compiling native code)
@@ -62,16 +61,6 @@ Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
   $ sudo apt-get -y install maven
 * Native libraries
   $ sudo apt-get -y install build-essential autoconf automake libtool cmake zlib1g-dev pkg-config libssl-dev libsasl2-dev
-* ProtocolBuffer 3.7.1 (required)
-  $ mkdir -p /opt/protobuf-3.7-src \
-        && curl -L -s -S \
-          https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-          -o /opt/protobuf-3.7.1.tar.gz \
-        && tar xzf /opt/protobuf-3.7.1.tar.gz --strip-components 1 -C /opt/protobuf-3.7-src \
-        && cd /opt/protobuf-3.7-src \
-        && ./configure\
-        && make install \
-        && rm -rf /opt/protobuf-3.7-src
 
 Optional packages:
 
@@ -310,16 +299,6 @@ level once; and then work from the submodule. Keep in mind that SNAPSHOTs
 time out after a while, using the Maven '-nsu' will stop Maven from trying
 to update SNAPSHOTs from external repos.
 
-----------------------------------------------------------------------------------
-Protocol Buffer compiler
-
-The version of Protocol Buffer compiler, protoc, must match the version of the
-protobuf JAR.
-
-If you have multiple versions of protoc in your system, you can set in your
-build shell the HADOOP_PROTOC_PATH environment variable to point to the one you
-want to use for the Hadoop build. If you don't define this environment variable,
-protoc is looked up in the PATH.
 ----------------------------------------------------------------------------------
 Importing projects to eclipse
 
@@ -405,15 +384,6 @@ Installing required dependencies for clean install of macOS 10.14:
 * Install native libraries, only openssl is required to compile native code,
 you may optionally install zlib, lz4, etc.
   $ brew install openssl
-* Protocol Buffers 3.7.1 (required)
-  $ wget https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz
-  $ mkdir -p protobuf-3.7 && tar zxvf protobuf-java-3.7.1.tar.gz --strip-components 1 -C protobuf-3.7
-  $ cd protobuf-3.7
-  $ ./configure
-  $ make
-  $ make check
-  $ make install
-  $ protoc --version
 
 Note that building Hadoop 3.1.1/3.1.2/3.2.0 native code from source is broken
 on macOS. For 3.1.1/3.1.2, you need to manually backport YARN-8622. For 3.2.0,
@@ -439,7 +409,6 @@ Requirements:
 * Windows System
 * JDK 1.8
 * Maven 3.0 or later
-* ProtocolBuffer 3.7.1
 * CMake 3.1 or newer
 * Visual Studio 2010 Professional or Higher
 * Windows SDK 8.1 (if building CPU rate control for the container executor)

dev-support/docker/Dockerfile

@@ -105,23 +105,6 @@ RUN mkdir -p /opt/cmake \
 ENV CMAKE_HOME /opt/cmake
 ENV PATH "${PATH}:/opt/cmake/bin"
 
-######
-# Install Google Protobuf 3.7.1 (2.6.0 ships with Xenial)
-######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/protobuf-src \
-    && curl -L -s -S \
-      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-      -o /opt/protobuf.tar.gz \
-    && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
-    && cd /opt/protobuf-src \
-    && ./configure --prefix=/opt/protobuf \
-    && make install \
-    && cd /root \
-    && rm -rf /opt/protobuf-src
-ENV PROTOBUF_HOME /opt/protobuf
-ENV PATH "${PATH}:/opt/protobuf/bin"
-
 ######
 # Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
 ######
@@ -207,7 +190,7 @@ ENV MAVEN_OPTS -Xms256m -Xmx1536m
 ###
 
 # Hugo static website generator (for new hadoop site and Ozone docs)
-RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.30.2/hugo_0.30.2_Linux-64bit.deb \
+RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.58.3/hugo_0.58.3_Linux-64bit.deb \
     && dpkg --install hugo.deb \
     && rm hugo.deb
 

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java

@@ -41,6 +41,7 @@
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
 import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
@@ -56,7 +57,6 @@
 import org.apache.ratis.retry.RetryPolicy;
 import org.apache.ratis.rpc.RpcType;
 import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
 import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
 import org.apache.ratis.util.TimeDuration;
 import org.slf4j.Logger;
@@ -219,39 +219,16 @@ private CompletableFuture<RaftClientReply> sendRequestAsync(
     try (Scope scope = GlobalTracer.get()
         .buildSpan("XceiverClientRatis." + request.getCmdType().name())
         .startActive(true)) {
-      ContainerCommandRequestProto finalPayload =
-          ContainerCommandRequestProto.newBuilder(request)
-              .setTraceID(TracingUtil.exportCurrentSpan())
-              .build();
-      boolean isReadOnlyRequest = HddsUtils.isReadOnly(finalPayload);
-      ByteString byteString = finalPayload.toByteString();
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("sendCommandAsync {} {}", isReadOnlyRequest,
-            sanitizeForDebug(finalPayload));
+      final ContainerCommandRequestMessage message
+          = ContainerCommandRequestMessage.toMessage(
+              request, TracingUtil.exportCurrentSpan());
+      if (HddsUtils.isReadOnly(request)) {
+        LOG.debug("sendCommandAsync ReadOnly {}", message);
+        return getClient().sendReadOnlyAsync(message);
+      } else {
+        LOG.debug("sendCommandAsync {}", message);
+        return getClient().sendAsync(message);
       }
-      return isReadOnlyRequest ?
-          getClient().sendReadOnlyAsync(() -> byteString) :
-          getClient().sendAsync(() -> byteString);
-    }
-  }
-
-  private ContainerCommandRequestProto sanitizeForDebug(
-      ContainerCommandRequestProto request) {
-    switch (request.getCmdType()) {
-    case PutSmallFile:
-      return request.toBuilder()
-          .setPutSmallFile(request.getPutSmallFile().toBuilder()
-              .clearData()
-          )
-          .build();
-    case WriteChunk:
-      return request.toBuilder()
-          .setWriteChunk(request.getWriteChunk().toBuilder()
-              .clearData()
-          )
-          .build();
-    default:
-      return request;
     }
   }
 

hadoop-hdds/common/dev-support/findbugsExcludeFile.xml

@@ -25,4 +25,9 @@
     <Class name="org.apache.hadoop.hdds.cli.GenericCli"></Class>
     <Bug pattern="DM_EXIT" />
   </Match>
+  <Match>
+    <Class name="org.apache.hadoop.ozone.common.ChecksumByteBuffer$CrcIntTable" />
+    <Method name="update" />
+    <Bug pattern="SF_SWITCH_FALLTHROUGH,SF_SWITCH_NO_DEFAULT" />
+  </Match>
 </FindBugsFilter>

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java

@@ -16,22 +16,29 @@
  */
 package org.apache.hadoop.hdds.protocolPB;
 
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.function.Consumer;
+
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
 import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCACertificateRequestProto;
 import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
 import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto.Builder;
 import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
-import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityRequest;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityRequest.Builder;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityResponse;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.Type;
+import org.apache.hadoop.hdds.tracing.TracingUtil;
 import org.apache.hadoop.ipc.ProtobufHelper;
 import org.apache.hadoop.ipc.ProtocolTranslator;
 import org.apache.hadoop.ipc.RPC;
 
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
 import static org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetOMCertRequestProto;
 
 /**
@@ -52,6 +59,28 @@ public SCMSecurityProtocolClientSideTranslatorPB(
     this.rpcProxy = rpcProxy;
   }
 
+  /**
+   * Helper method to wrap the request and send the message.
+   */
+  private SCMSecurityResponse submitRequest(
+      SCMSecurityProtocolProtos.Type type,
+      Consumer<Builder> builderConsumer) throws IOException {
+    final SCMSecurityResponse response;
+    try {
+
+      Builder builder = SCMSecurityRequest.newBuilder()
+          .setCmdType(type)
+          .setTraceID(TracingUtil.exportCurrentSpan());
+      builderConsumer.accept(builder);
+      SCMSecurityRequest wrapper = builder.build();
+
+      response = rpcProxy.submitRequest(NULL_RPC_CONTROLLER, wrapper);
+    } catch (ServiceException ex) {
+      throw ProtobufHelper.getRemoteException(ex);
+    }
+    return response;
+  }
+
   /**
    * Closes this stream and releases any system resources associated
    * with it. If the stream is already closed then invoking this
@@ -87,8 +116,8 @@ public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
   /**
    * Get SCM signed certificate for OM.
    *
-   * @param omDetails       - OzoneManager Details.
-   * @param certSignReq     - Certificate signing request.
+   * @param omDetails   - OzoneManager Details.
+   * @param certSignReq - Certificate signing request.
    * @return byte[]         - SCM signed certificate.
    */
   @Override
@@ -100,64 +129,61 @@ public String getOMCertificate(OzoneManagerDetailsProto omDetails,
   /**
    * Get SCM signed certificate for OM.
    *
-   * @param omDetails       - OzoneManager Details.
-   * @param certSignReq     - Certificate signing request.
+   * @param omDetails   - OzoneManager Details.
+   * @param certSignReq - Certificate signing request.
    * @return byte[]         - SCM signed certificate.
    */
   public SCMGetCertResponseProto getOMCertChain(
       OzoneManagerDetailsProto omDetails, String certSignReq)
       throws IOException {
-    SCMGetOMCertRequestProto.Builder builder = SCMGetOMCertRequestProto
+    SCMGetOMCertRequestProto request = SCMGetOMCertRequestProto
         .newBuilder()
         .setCSR(certSignReq)
-        .setOmDetails(omDetails);
-    try {
-      return rpcProxy.getOMCertificate(NULL_RPC_CONTROLLER, builder.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
+        .setOmDetails(omDetails)
+        .build();
+    return submitRequest(Type.GetOMCertificate,
+        builder -> builder.setGetOMCertRequest(request))
+        .getGetCertResponseProto();
   }
 
   /**
    * Get SCM signed certificate with given serial id. Throws exception if
    * certificate is not found.
    *
-   * @param certSerialId    - Certificate serial id.
+   * @param certSerialId - Certificate serial id.
    * @return string         - pem encoded certificate.
    */
   @Override
   public String getCertificate(String certSerialId) throws IOException {
-    Builder builder = SCMGetCertificateRequestProto
+    SCMGetCertificateRequestProto request = SCMGetCertificateRequestProto
         .newBuilder()
-        .setCertSerialId(certSerialId);
-    try {
-      return rpcProxy.getCertificate(NULL_RPC_CONTROLLER, builder.build())
-          .getX509Certificate();
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
+        .setCertSerialId(certSerialId)
+        .build();
+    return submitRequest(Type.GetCertificate,
+        builder -> builder.setGetCertificateRequest(request))
+        .getGetCertResponseProto()
+        .getX509Certificate();
   }
 
   /**
    * Get SCM signed certificate for Datanode.
    *
-   * @param dnDetails       - Datanode Details.
-   * @param certSignReq     - Certificate signing request.
+   * @param dnDetails   - Datanode Details.
+   * @param certSignReq - Certificate signing request.
    * @return byte[]         - SCM signed certificate.
    */
   public SCMGetCertResponseProto getDataNodeCertificateChain(
       DatanodeDetailsProto dnDetails, String certSignReq)
       throws IOException {
-    SCMGetDataNodeCertRequestProto.Builder builder =
+
+    SCMGetDataNodeCertRequestProto request =
         SCMGetDataNodeCertRequestProto.newBuilder()
             .setCSR(certSignReq)
-            .setDatanodeDetails(dnDetails);
-    try {
-      return rpcProxy.getDataNodeCertificate(NULL_RPC_CONTROLLER,
-          builder.build());
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
+            .setDatanodeDetails(dnDetails)
+            .build();
+    return submitRequest(Type.GetDataNodeCertificate,
+        builder -> builder.setGetDataNodeCertRequest(request))
+        .getGetCertResponseProto();
   }
 
   /**
@@ -169,12 +195,10 @@ public SCMGetCertResponseProto getDataNodeCertificateChain(
   public String getCACertificate() throws IOException {
     SCMGetCACertificateRequestProto protoIns = SCMGetCACertificateRequestProto
         .getDefaultInstance();
-    try {
-      return rpcProxy.getCACertificate(NULL_RPC_CONTROLLER, protoIns)
-          .getX509Certificate();
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
+    return submitRequest(Type.GetCACertificate,
+        builder -> builder.setGetCACertificateRequest(protoIns))
+        .getGetCertResponseProto().getX509Certificate();
+
   }
 
   /**