HADOOP-16922. ABFS: Change User-Agent header (apache#1938)

Contributed by Bilahari T H.

Author: bilaharith

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java

@@ -168,9 +168,17 @@ public class AbfsConfiguration{
   private boolean enableAutoThrottling;
 
   @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_USER_AGENT_PREFIX_KEY,
-      DefaultValue = "")
+      DefaultValue = DEFAULT_FS_AZURE_USER_AGENT_PREFIX)
   private String userAgentId;
 
+  @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLUSTER_NAME,
+      DefaultValue = DEFAULT_VALUE_UNKNOWN)
+  private String clusterName;
+
+  @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLUSTER_TYPE,
+      DefaultValue = DEFAULT_VALUE_UNKNOWN)
+  private String clusterType;
+
   @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_DELEGATION_TOKEN,
       DefaultValue = DEFAULT_ENABLE_DELEGATION_TOKEN)
   private boolean enableDelegationToken;
@@ -479,6 +487,14 @@ public String getCustomUserAgentPrefix() {
     return this.userAgentId;
   }
 
+  public String getClusterName() {
+    return this.clusterName;
+  }
+
+  public String getClusterType() {
+    return this.clusterType;
+  }
+
   public DelegatingSSLSocketFactory.SSLChannelMode getPreferredSSLFactoryOption() {
     return getEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DEFAULT_FS_AZURE_SSL_CHANNEL_MODE);
   }

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java

@@ -42,10 +42,13 @@ public final class AbfsHttpConstants {
   public static final String DEFAULT_TIMEOUT = "90";
   public static final String TOKEN_VERSION = "2";
 
+  public static final String JAVA_VENDOR = "java.vendor";
   public static final String JAVA_VERSION = "java.version";
   public static final String OS_NAME = "os.name";
   public static final String OS_VERSION = "os.version";
+  public static final String OS_ARCH = "os.arch";
 
+  public static final String APN_VERSION = "APN/1.0";
   public static final String CLIENT_VERSION = "Azure Blob FS/" + VersionInfo.getVersion();
 
   // Abfs Http Verb

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java

@@ -62,6 +62,8 @@ public final class ConfigurationKeys {
    *  Default value of this config is true. **/
   public static final String FS_AZURE_DISABLE_OUTPUTSTREAM_FLUSH = "fs.azure.disable.outputstream.flush";
   public static final String FS_AZURE_USER_AGENT_PREFIX_KEY = "fs.azure.user.agent.prefix";
+  public static final String FS_AZURE_CLUSTER_NAME = "fs.azure.cluster.name";
+  public static final String FS_AZURE_CLUSTER_TYPE = "fs.azure.cluster.type";
   public static final String FS_AZURE_SSL_CHANNEL_MODE_KEY = "fs.azure.ssl.channel.mode";
   /** Provides a config to enable/disable the checkAccess API.
    *  By default this will be

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java

@@ -22,6 +22,8 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
 
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING;
+
 /**
  * Responsible to keep all the Azure Blob File System related configurations.
  */
@@ -71,5 +73,8 @@ public final class FileSystemConfigurations {
   public static final boolean DEFAULT_ENABLE_CHECK_ACCESS = false;
   public static final boolean DEFAULT_ABFS_LATENCY_TRACK = false;
 
+  public static final String DEFAULT_FS_AZURE_USER_AGENT_PREFIX = EMPTY_STRING;
+  public static final String DEFAULT_VALUE_UNKNOWN = "UNKNOWN";
+
   private FileSystemConfigurations() {}
 }

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java

@@ -679,32 +679,60 @@ public AuthType getAuthType() {
 
   @VisibleForTesting
   String initializeUserAgent(final AbfsConfiguration abfsConfiguration,
-                             final String sslProviderName) {
+      final String sslProviderName) {
+
     StringBuilder sb = new StringBuilder();
-    sb.append("(JavaJRE ");
+
+    sb.append(APN_VERSION);
+    sb.append(SINGLE_WHITE_SPACE);
+    sb.append(CLIENT_VERSION);
+    sb.append(SINGLE_WHITE_SPACE);
+
+    sb.append("(");
+
+    sb.append(System.getProperty(JAVA_VENDOR)
+        .replaceAll(SINGLE_WHITE_SPACE, EMPTY_STRING));
+    sb.append(SINGLE_WHITE_SPACE);
+    sb.append("JavaJRE");
+    sb.append(SINGLE_WHITE_SPACE);
     sb.append(System.getProperty(JAVA_VERSION));
-    sb.append("; ");
-    sb.append(
-        System.getProperty(OS_NAME).replaceAll(SINGLE_WHITE_SPACE, EMPTY_STRING));
-    sb.append(" ");
+    sb.append(SEMICOLON);
+    sb.append(SINGLE_WHITE_SPACE);
+
+    sb.append(System.getProperty(OS_NAME)
+        .replaceAll(SINGLE_WHITE_SPACE, EMPTY_STRING));
+    sb.append(SINGLE_WHITE_SPACE);
     sb.append(System.getProperty(OS_VERSION));
-    if (sslProviderName != null && !sslProviderName.isEmpty()) {
-      sb.append("; ");
-      sb.append(sslProviderName);
-    }
-    String tokenProviderField =
-        ExtensionHelper.getUserAgentSuffix(tokenProvider, "");
-    if (!tokenProviderField.isEmpty()) {
-      sb.append("; ").append(tokenProviderField);
-    }
+    sb.append(FORWARD_SLASH);
+    sb.append(System.getProperty(OS_ARCH));
+    sb.append(SEMICOLON);
+
+    appendIfNotEmpty(sb, sslProviderName, true);
+    appendIfNotEmpty(sb,
+        ExtensionHelper.getUserAgentSuffix(tokenProvider, EMPTY_STRING), true);
+
+    sb.append(SINGLE_WHITE_SPACE);
+    sb.append(abfsConfiguration.getClusterName());
+    sb.append(FORWARD_SLASH);
+    sb.append(abfsConfiguration.getClusterType());
+
     sb.append(")");
-    final String userAgentComment = sb.toString();
-    String customUserAgentId = abfsConfiguration.getCustomUserAgentPrefix();
-    if (customUserAgentId != null && !customUserAgentId.isEmpty()) {
-      return String.format(Locale.ROOT, CLIENT_VERSION + " %s %s",
-          userAgentComment, customUserAgentId);
+
+    appendIfNotEmpty(sb, abfsConfiguration.getCustomUserAgentPrefix(), false);
+
+    return String.format(Locale.ROOT, sb.toString());
+  }
+
+  private void appendIfNotEmpty(StringBuilder sb, String regEx,
+      boolean shouldAppendSemiColon) {
+    if (regEx == null || regEx.trim().isEmpty()) {
+      return;
+    }
+    sb.append(SINGLE_WHITE_SPACE);
+    sb.append(regEx);
+    if (shouldAppendSemiColon) {
+      sb.append(SEMICOLON);
     }
-    return String.format(Locale.ROOT, CLIENT_VERSION + " %s", userAgentComment);
   }
 
   @VisibleForTesting