Merge branch 'apache:trunk' into YARN-11350-V2

Author: slfan1989

LICENSE-binary

@@ -324,7 +324,7 @@ org.apache.htrace:htrace-core:3.1.0-incubating
 org.apache.htrace:htrace-core4:4.1.0-incubating
 org.apache.httpcomponents:httpclient:4.5.6
 org.apache.httpcomponents:httpcore:4.4.10
-org.apache.kafka:kafka-clients:2.8.1
+org.apache.kafka:kafka-clients:2.8.2
 org.apache.kerby:kerb-admin:2.0.2
 org.apache.kerby:kerb-client:2.0.2
 org.apache.kerby:kerb-common:2.0.2

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/util/PlatformName.java

@@ -18,6 +18,10 @@
 
 package org.apache.hadoop.util;
 
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Arrays;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
@@ -33,21 +37,71 @@ public class PlatformName {
    * per the java-vm.
    */
   public static final String PLATFORM_NAME =
-      (System.getProperty("os.name").startsWith("Windows")
-      ? System.getenv("os") : System.getProperty("os.name"))
-      + "-" + System.getProperty("os.arch")
-      + "-" + System.getProperty("sun.arch.data.model");
+      (System.getProperty("os.name").startsWith("Windows") ?
+      System.getenv("os") : System.getProperty("os.name"))
+      + "-" + System.getProperty("os.arch") + "-"
+      + System.getProperty("sun.arch.data.model");
 
   /**
    * The java vendor name used in this platform.
    */
   public static final String JAVA_VENDOR_NAME = System.getProperty("java.vendor");
 
+  /**
+   * Define a system class accessor that is open to changes in underlying implementations
+   * of the system class loader modules.
+   */
+  private static final class SystemClassAccessor extends ClassLoader {
+    public Class<?> getSystemClass(String className) throws ClassNotFoundException {
+      return findSystemClass(className);
+    }
+  }
+
   /**
    * A public static variable to indicate the current java vendor is
-   * IBM java or not.
+   * IBM and the type is Java Technology Edition which provides its
+   * own implementations of many security packages and Cipher suites.
+   * Note that these are not provided in Semeru runtimes:
+   * See https://developer.ibm.com/languages/java/semeru-runtimes for details.
    */
-  public static final boolean IBM_JAVA = JAVA_VENDOR_NAME.contains("IBM");
+  public static final boolean IBM_JAVA = JAVA_VENDOR_NAME.contains("IBM") &&
+      hasIbmTechnologyEditionModules();
+
+  private static boolean hasIbmTechnologyEditionModules() {
+    return Arrays.asList(
+        "com.ibm.security.auth.module.JAASLoginModule",
+        "com.ibm.security.auth.module.Win64LoginModule",
+        "com.ibm.security.auth.module.NTLoginModule",
+        "com.ibm.security.auth.module.AIX64LoginModule",
+        "com.ibm.security.auth.module.LinuxLoginModule",
+        "com.ibm.security.auth.module.Krb5LoginModule"
+    ).stream().anyMatch((module) -> isSystemClassAvailable(module));
+  }
+
+  /**
+   * In rare cases where different behaviour is performed based on the JVM vendor
+   * this method should be used to test for a unique JVM class provided by the
+   * vendor rather than using the vendor method. For example if on JVM provides a
+   * different Kerberos login module testing for that login module being loadable
+   * before configuring to use it is preferable to using the vendor data.
+   *
+   * @param className the name of a class in the JVM to test for
+   * @return true if the class is available, false otherwise.
+   */
+  private static boolean isSystemClassAvailable(String className) {
+    return AccessController.doPrivileged((PrivilegedAction<Boolean>) () -> {
+      try {
+        // Using ClassLoader.findSystemClass() instead of
+        // Class.forName(className, false, null) because Class.forName with a null
+        // ClassLoader only looks at the boot ClassLoader with Java 9 and above
+        // which doesn't look at all the modules available to the findSystemClass.
+        new SystemClassAccessor().getSystemClass(className);
+        return true;
+      } catch (Exception ignored) {
+        return false;
+      }
+    });
+  }
 
   public static void main(String[] args) {
     System.out.println(PLATFORM_NAME);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java

@@ -60,7 +60,6 @@ public AvroFSInput(final FileContext fc, final Path p) throws IOException {
             FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)
         .withFileStatus(status)
         .build());
-    fc.open(p);
   }
 
   @Override

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java

@@ -25,7 +25,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.util.PlatformName.JAVA_VENDOR_NAME;
+import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.HttpsURLConnection;
@@ -102,11 +102,11 @@ public enum Mode { CLIENT, SERVER }
       "ssl.server.exclude.cipher.list";
 
   public static final String KEY_MANAGER_SSLCERTIFICATE =
-      JAVA_VENDOR_NAME.contains("IBM") ? "ibmX509" :
+      IBM_JAVA ? "ibmX509" :
           KeyManagerFactory.getDefaultAlgorithm();
 
   public static final String TRUST_MANAGER_SSLCERTIFICATE =
-      JAVA_VENDOR_NAME.contains("IBM") ? "ibmX509" :
+      IBM_JAVA ? "ibmX509" :
           TrustManagerFactory.getDefaultAlgorithm();
 
   public static final String KEYSTORES_FACTORY_CLASS_KEY =

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java

@@ -108,7 +108,7 @@ static URL[] constructUrlsFromClasspath(String classpath)
       throws MalformedURLException {
     List<URL> urls = new ArrayList<URL>();
     for (String element : classpath.split(File.pathSeparator)) {
-      if (element.endsWith("/*")) {
+      if (element.endsWith(File.separator + "*")) {
         List<Path> jars = FileUtil.getJarsInDirectory(element);
         if (!jars.isEmpty()) {
           for (Path jar: jars) {

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -2168,9 +2168,8 @@ The switch to turn S3A auditing on or off.
 
 <property>
   <name>fs.azure.enable.readahead</name>
-  <value>false</value>
-  <description>Disable readahead/prefetching in AbfsInputStream.
-    See HADOOP-18521</description>
+  <value>true</value>
+  <description>Enabled readahead/prefetching in AbfsInputStream.</description>
 </property>
 
 <property>

hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md

@@ -22,7 +22,17 @@ Purpose
 
 This document describes how to install and configure Hadoop clusters ranging from a few nodes to extremely large clusters with thousands of nodes. To play with Hadoop, you may first want to install it on a single machine (see [Single Node Setup](./SingleCluster.html)).
 
-This document does not cover advanced topics such as [Security](./SecureMode.html) or High Availability.
+This document does not cover advanced topics such as High Availability.
+
+*Important*: all production Hadoop clusters use Kerberos to authenticate callers
+and secure access to HDFS data as well as restriction access to computation
+services (YARN etc.). 
+
+These instructions do not cover integration with any Kerberos services,
+-everyone bringing up a production cluster should include connecting to their
+organisation's Kerberos infrastructure as a key part of the deployment.
+
+See [Security](./SecureMode.html) for details on how to secure a cluster. 
 
 Prerequisites
 -------------

hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm

@@ -26,15 +26,22 @@ Purpose
 
 This document describes how to set up and configure a single-node Hadoop installation so that you can quickly perform simple operations using Hadoop MapReduce and the Hadoop Distributed File System (HDFS).
 
+
+*Important*: all production Hadoop clusters use Kerberos to authenticate callers
+and secure access to HDFS data as well as restriction access to computation
+services (YARN etc.). 
+
+These instructions do not cover integration with any Kerberos services,
+-everyone bringing up a production cluster should include connecting to their
+organisation's Kerberos infrastructure as a key part of the deployment.
+
 Prerequisites
 -------------
 
 $H3 Supported Platforms
 
 * GNU/Linux is supported as a development and production platform. Hadoop has been demonstrated on GNU/Linux clusters with 2000 nodes.
 
-* Windows is also a supported platform but the followings steps are for Linux only. To set up Hadoop on Windows, see [wiki page](http://wiki.apache.org/hadoop/Hadoop2OnWindows).
-
 $H3 Required Software
 
 Required software for Linux include:

hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java

@@ -38,8 +38,35 @@
 import java.util.Arrays;
 
 public class TestMiniKdc extends KerberosSecurityTestcase {
-  private static final boolean IBM_JAVA = System.getProperty("java.vendor")
-      .contains("IBM");
+  private static final boolean IBM_JAVA = shouldUseIbmPackages();
+  // duplicated to avoid cycles in the build
+  private static boolean shouldUseIbmPackages() {
+    final List<String> ibmTechnologyEditionSecurityModules = Arrays.asList(
+        "com.ibm.security.auth.module.JAASLoginModule",
+        "com.ibm.security.auth.module.Win64LoginModule",
+        "com.ibm.security.auth.module.NTLoginModule",
+        "com.ibm.security.auth.module.AIX64LoginModule",
+        "com.ibm.security.auth.module.LinuxLoginModule",
+        "com.ibm.security.auth.module.Krb5LoginModule"
+    );
+
+    if (System.getProperty("java.vendor").contains("IBM")) {
+      return ibmTechnologyEditionSecurityModules
+          .stream().anyMatch((module) -> isSystemClassAvailable(module));
+    }
+
+    return false;
+  }
+
+  private static boolean isSystemClassAvailable(String className) {
+    try {
+      Class.forName(className);
+      return true;
+    } catch (Exception ignored) {
+      return false;
+    }
+  }
+
   @Test
   public void testMiniKdcStart() {
     MiniKdc kdc = getKdc();
@@ -117,9 +144,9 @@ public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
       options.put("debug", "true");
 
       return new AppConfigurationEntry[]{
-              new AppConfigurationEntry(getKrb5LoginModuleName(),
-                      AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
-                      options)};
+          new AppConfigurationEntry(getKrb5LoginModuleName(),
+                AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
+                options)};
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java

@@ -72,6 +72,8 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STARTUP_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT;
@@ -353,6 +355,7 @@ public class DataNode extends ReconfigurableBase
               DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
               DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY,
               DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY,
+              DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY,
               FS_DU_INTERVAL_KEY,
               FS_GETSPACEUSED_JITTER_KEY,
               FS_GETSPACEUSED_CLASSNAME));
@@ -699,6 +702,7 @@ public String reconfigurePropertyImpl(String property, String newVal)
     case DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY:
     case DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY:
     case DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY:
+    case DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY:
       return reconfSlowDiskParameters(property, newVal);
     case FS_DU_INTERVAL_KEY:
     case FS_GETSPACEUSED_JITTER_KEY:
@@ -877,6 +881,12 @@ private String reconfSlowDiskParameters(String property, String newVal)
             Long.parseLong(newVal));
         result = Long.toString(threshold);
         diskMetrics.setLowThresholdMs(threshold);
+      } else if (property.equals(DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY)) {
+        checkNotNull(diskMetrics, "DataNode disk stats may be disabled.");
+        int maxSlowDisksToExclude = (newVal == null ?
+            DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_DEFAULT : Integer.parseInt(newVal));
+        result = Integer.toString(maxSlowDisksToExclude);
+        diskMetrics.setMaxSlowDisksToExclude(maxSlowDisksToExclude);
       }
       LOG.info("RECONFIGURE* changed {} to {}", property, newVal);
       return result;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java

@@ -80,7 +80,7 @@ public class DataNodeDiskMetrics {
   /**
    * The number of slow disks that needs to be excluded.
    */
-  private int maxSlowDisksToExclude;
+  private volatile int maxSlowDisksToExclude;
   /**
    * List of slow disks that need to be excluded.
    */
@@ -274,6 +274,14 @@ public List<String> getSlowDisksToExclude() {
     return slowDisksToExclude;
   }
 
+  public int getMaxSlowDisksToExclude() {
+    return maxSlowDisksToExclude;
+  }
+
+  public void setMaxSlowDisksToExclude(int maxSlowDisksToExclude) {
+    this.maxSlowDisksToExclude = maxSlowDisksToExclude;
+  }
+
   public void setLowThresholdMs(long thresholdMs) {
     Preconditions.checkArgument(thresholdMs > 0,
         DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY + " should be larger than 0");

hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js

@@ -45,6 +45,7 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
@@ -636,13 +637,15 @@ public void testSlowDiskParameters() throws ReconfigurationException, IOExceptio
       String[] slowDisksParameters2 = {
           DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY,
           DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY,
-          DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY};
+          DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY,
+          DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY};
       for (String parameter : slowDisksParameters2) {
         dn.reconfigureProperty(parameter, "99");
       }
       // Assert diskMetrics.
       assertEquals(99, dn.getDiskMetrics().getMinOutlierDetectionDisks());
       assertEquals(99, dn.getDiskMetrics().getLowThresholdMs());
+      assertEquals(99, dn.getDiskMetrics().getMaxSlowDisksToExclude());
       // Assert dnConf.
       assertTrue(dn.getDnConf().diskStatsEnabled);
       // Assert profilingEventHook.
@@ -673,12 +676,16 @@ public void testSlowDiskParameters() throws ReconfigurationException, IOExceptio
       dn.reconfigureProperty(DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY, "1");
       dn.reconfigureProperty(DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY, null);
       dn.reconfigureProperty(DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY, null);
+      dn.reconfigureProperty(DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY, null);
       assertEquals(String.format("expect %s is not configured",
           DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY), null,
           dn.getConf().get(DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY));
       assertEquals(String.format("expect %s is not configured",
           DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY), null,
           dn.getConf().get(DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY));
+      assertEquals(String.format("expect %s is not configured",
+          DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY), null,
+          dn.getConf().get(DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY));
       assertEquals(DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT,
           dn.getDiskMetrics().getSlowDiskDetector().getMinOutlierDetectionNodes());
       assertEquals(DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT,

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java

@@ -345,7 +345,7 @@ public void testDataNodeGetReconfigurableProperties() throws IOException, Interr
     final List<String> outs = Lists.newArrayList();
     final List<String> errs = Lists.newArrayList();
     getReconfigurableProperties("datanode", address, outs, errs);
-    assertEquals(19, outs.size());
+    assertEquals(20, outs.size());
     assertEquals(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, outs.get(1));
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java

@@ -50,7 +50,7 @@
     <!-- Version number for xerces used by JDiff -->
     <xerces.jdiff.version>2.12.2</xerces.jdiff.version>
 
-    <kafka.version>2.8.1</kafka.version>
+    <kafka.version>2.8.2</kafka.version>
 
     <commons-daemon.version>1.0.13</commons-daemon.version>