Hadoop 16438

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -3351,6 +3351,17 @@
     </description>
   </property>
 
+  <property>
+    <name>adl.ssl.channel.mode</name>
+    <value></value>
+    <description>
+      When OpenSSL - SSL socket connections are created in OpenSSL mode.
+      When Default_JSSE - SSL socket connections are created in the default JSSE mode.
+      When Default (default) - SSL socket connections are attempted with OpenSSL
+      and will fallback to Default_JSSE mode if OpenSSL is not available at runtime.
+    </description>
+  </property>
+
   <!-- Azure Data Lake File System Configurations Ends Here-->
 
   <property>

hadoop-tools/hadoop-azure-datalake/pom.xml

@@ -33,7 +33,7 @@
     <minimalJsonVersion>0.9.1</minimalJsonVersion>
     <file.encoding>UTF-8</file.encoding>
     <downloadSources>true</downloadSources>
-    <azure.data.lake.store.sdk.version>2.3.3</azure.data.lake.store.sdk.version>
+    <azure.data.lake.store.sdk.version>2.3.6</azure.data.lake.store.sdk.version>
   </properties>
   <build>
     <plugins>

hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java

@@ -106,6 +106,7 @@ public final class AdlConfKeys {
       "adl.feature.ownerandgroup.enableupn";
   static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
   public static final String ADL_HTTP_TIMEOUT = "adl.http.timeout";
+  public static final String ADL_SSL_CHANNEL_MODE = "adl.ssl.channel.mode";
 
   public static void addDeprecatedKeys() {
     Configuration.addDeprecations(new DeprecationDelta[]{

hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java

@@ -203,6 +203,10 @@ public void initialize(URI storeUri, Configuration originalConf)
       LOG.info("No valid ADL SDK timeout configured: using SDK default.");
     }
 
+    String sslChannelMode = conf.get(ADL_SSL_CHANNEL_MODE,
+        "Default");
+    options.setSSLChannelMode(sslChannelMode);
+
     adlClient.setOptions(options);
 
     boolean trackLatency = conf

hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md

@@ -153,3 +153,13 @@ addressed by lowering the timeout used by the SDK.  A lower timeout at the
 storage layer may allow more retries to be attempted and actually increase
 the likelihood of success before hitting the framework's timeout, as attempts
 that may ultimately fail will fail faster.
+
+## SSL Socket Channel Mode
+
+ADL SDK will by default attempt to create secure socket connections over
+OpenSSL as they provide significant performance improvements over Https. If
+there are runtime issues, SDK will default connections over Default_JSSE. This
+can be overridden with the hadoop property `adl.ssl.channel.mode`. Possible
+values for this config are OpenSSL, Default_JSSE and Default (default).
+Setting the config to OpenSSL or Default_JSSE will try the connection to
+only that mode.

hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java

@@ -19,6 +19,7 @@
 
 package org.apache.hadoop.fs.adl.live;
 
+import com.microsoft.azure.datalake.store.SSLSocketFactoryEx.SSLChannelMode;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.adl.AdlFileSystem;
 import org.junit.Assert;
@@ -29,6 +30,7 @@
 import java.net.URISyntaxException;
 
 import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_HTTP_TIMEOUT;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_SSL_CHANNEL_MODE;
 
 /**
  * Tests interactions with SDK and ensures configuration is having the desired
@@ -53,7 +55,6 @@ public void testDefaultTimeout() throws IOException {
 
     // Skip this test if we can't get a real FS
     Assume.assumeNotNull(fs);
-
     effectiveTimeout = fs.getAdlClient().getDefaultTimeout();
     Assert.assertFalse("A negative timeout is not supposed to take effect",
         effectiveTimeout < 0);
@@ -74,4 +75,67 @@ public void testDefaultTimeout() throws IOException {
 
     // The default value may vary by SDK, so that value is not tested here.
   }
+
+  @Test
+  public void testSSLChannelMode() throws IOException {
+    AdlFileSystem fs = null;
+    Configuration conf = null;
+
+    conf = AdlStorageConfiguration.getConfiguration();
+    conf.set(ADL_SSL_CHANNEL_MODE, "OpenSSl");
+    try {
+      fs = (AdlFileSystem)
+          (AdlStorageConfiguration.createStorageConnector(conf));
+    } catch (URISyntaxException e) {
+      throw new IllegalStateException("ADL FileSystem initialization failed. "
+          + "Please check test.fs.adl.name property.", e);
+    }
+
+    SSLChannelMode sslChannelMode = fs.getAdlClient().getSSLChannelMode();
+    Assert.assertTrue("Channel mode needs to be OpenSSL",
+        sslChannelMode == SSLChannelMode.OpenSSL);
+
+    conf = AdlStorageConfiguration.getConfiguration();
+    conf.set(ADL_SSL_CHANNEL_MODE, "Default_JSE");
+    try {
+      fs = (AdlFileSystem)
+          (AdlStorageConfiguration.createStorageConnector(conf));
+    } catch (URISyntaxException e) {
+      throw new IllegalStateException("Can not initialize ADL FileSystem. "
+          + "Please check test.fs.adl.name property.", e);
+    }
+
+    sslChannelMode = fs.getAdlClient().getSSLChannelMode();
+    Assert.assertTrue("Channel mode needs to be Default_JSE",
+        sslChannelMode == SSLChannelMode.Default_JSE);
+
+    conf = AdlStorageConfiguration.getConfiguration();
+    conf.set(ADL_SSL_CHANNEL_MODE, "Default");
+    try {
+      fs = (AdlFileSystem)
+          (AdlStorageConfiguration.createStorageConnector(conf));
+    } catch (URISyntaxException e) {
+      throw new IllegalStateException("Can not initialize ADL FileSystem. "
+          + "Please check test.fs.adl.name property.", e);
+    }
+
+    sslChannelMode = fs.getAdlClient().getSSLChannelMode();
+    Assert.assertTrue("Channel mode needs to be Default",
+        sslChannelMode == SSLChannelMode.Default);
+
+    conf = AdlStorageConfiguration.getConfiguration();
+    conf.set(ADL_SSL_CHANNEL_MODE, "Invalid");
+    try {
+      fs = (AdlFileSystem)
+          (AdlStorageConfiguration.createStorageConnector(conf));
+    } catch (URISyntaxException e) {
+      throw new IllegalStateException("Can not initialize ADL FileSystem. "
+          + "Please check test.fs.adl.name property.", e);
+    }
+
+    sslChannelMode = fs.getAdlClient().getSSLChannelMode();
+    Assert.assertTrue("Channel mode needs to be Default when adl.ssl"
+            + ".channel.mode config is missing or is invalid",
+        sslChannelMode == SSLChannelMode.Default);
+  }
 }