Merge branch 'trunk' into YARN-11509

slfan1989 · web-flow · commit fda62cf51efb · 2023-07-04T13:27:17.000+08:00
diff --git a/LICENSE-binary b/LICENSE-binary
@@ -259,36 +259,36 @@ io.grpc:grpc-netty:1.26.0
 io.grpc:grpc-protobuf:1.26.0
 io.grpc:grpc-protobuf-lite:1.26.0
 io.grpc:grpc-stub:1.26.0
-io.netty:netty-all:4.1.77.Final
-io.netty:netty-buffer:4.1.77.Final
-io.netty:netty-codec:4.1.77.Final
-io.netty:netty-codec-dns:4.1.77.Final
-io.netty:netty-codec-haproxy:4.1.77.Final
-io.netty:netty-codec-http:4.1.77.Final
-io.netty:netty-codec-http2:4.1.77.Final
-io.netty:netty-codec-memcache:4.1.77.Final
-io.netty:netty-codec-mqtt:4.1.77.Final
-io.netty:netty-codec-redis:4.1.77.Final
-io.netty:netty-codec-smtp:4.1.77.Final
-io.netty:netty-codec-socks:4.1.77.Final
-io.netty:netty-codec-stomp:4.1.77.Final
-io.netty:netty-codec-xml:4.1.77.Final
-io.netty:netty-common:4.1.77.Final
-io.netty:netty-handler:4.1.77.Final
-io.netty:netty-handler-proxy:4.1.77.Final
-io.netty:netty-resolver:4.1.77.Final
-io.netty:netty-resolver-dns:4.1.77.Final
-io.netty:netty-transport:4.1.77.Final
-io.netty:netty-transport-rxtx:4.1.77.Final
-io.netty:netty-transport-sctp:4.1.77.Final
-io.netty:netty-transport-udt:4.1.77.Final
-io.netty:netty-transport-classes-epoll:4.1.77.Final
-io.netty:netty-transport-native-unix-common:4.1.77.Final
-io.netty:netty-transport-classes-kqueue:4.1.77.Final
-io.netty:netty-resolver-dns-classes-macos:4.1.77.Final
-io.netty:netty-transport-native-epoll:4.1.77.Final
-io.netty:netty-transport-native-kqueue:4.1.77.Final
-io.netty:netty-resolver-dns-native-macos:4.1.77.Final
+io.netty:netty-all:4.1.94.Final
+io.netty:netty-buffer:4.1.94.Final
+io.netty:netty-codec:4.1.94.Final
+io.netty:netty-codec-dns:4.1.94.Final
+io.netty:netty-codec-haproxy:4.1.94.Final
+io.netty:netty-codec-http:4.1.94.Final
+io.netty:netty-codec-http2:4.1.94.Final
+io.netty:netty-codec-memcache:4.1.94.Final
+io.netty:netty-codec-mqtt:4.1.94.Final
+io.netty:netty-codec-redis:4.1.94.Final
+io.netty:netty-codec-smtp:4.1.94.Final
+io.netty:netty-codec-socks:4.1.94.Final
+io.netty:netty-codec-stomp:4.1.94.Final
+io.netty:netty-codec-xml:4.1.94.Final
+io.netty:netty-common:4.1.94.Final
+io.netty:netty-handler:4.1.94.Final
+io.netty:netty-handler-proxy:4.1.94.Final
+io.netty:netty-resolver:4.1.94.Final
+io.netty:netty-resolver-dns:4.1.94.Final
+io.netty:netty-transport:4.1.94.Final
+io.netty:netty-transport-rxtx:4.1.94.Final
+io.netty:netty-transport-sctp:4.1.94.Final
+io.netty:netty-transport-udt:4.1.94.Final
+io.netty:netty-transport-classes-epoll:4.1.94.Final
+io.netty:netty-transport-native-unix-common:4.1.94.Final
+io.netty:netty-transport-classes-kqueue:4.1.94.Final
+io.netty:netty-resolver-dns-classes-macos:4.1.94.Final
+io.netty:netty-transport-native-epoll:4.1.94.Final
+io.netty:netty-transport-native-kqueue:4.1.94.Final
+io.netty:netty-resolver-dns-native-macos:4.1.94.Final
 io.opencensus:opencensus-api:0.12.3
 io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
 io.reactivex:rxjava:1.3.8
diff --git a/dev-support/git-jira-validation/README.md b/dev-support/git-jira-validation/README.md
@@ -68,8 +68,8 @@ The script also requires below inputs:
    Exact fixVersion that we would like to compare all Jira's fixVersions
    with. e.g for 3.3.2 release, it should be 3.3.2.
 
-3. JIRA Project Name:
-   The exact name of Project as case-sensitive e.g HADOOP / OZONE
+3. JIRA Project Name (default Project Name: HADOOP):
+   The exact name of Project as case-sensitive.
 
 4. Path of project's working dir with release branch checked-in:
    Path of project from where we want to compare git hashes from. Local fork
@@ -84,7 +84,7 @@ The script also requires below inputs:
 
 Example of script execution:
 ```
-JIRA Project Name (e.g HADOOP / OZONE etc): HADOOP
+JIRA Project Name (default: HADOOP): HADOOP
 First commit hash to start excluding commits from history: fa4915fdbbbec434ab41786cb17b82938a613f16
 Fix Version: 3.3.2
 Jira server url (default: https://issues.apache.org/jira):
diff --git a/dev-support/git-jira-validation/git_jira_fix_version_check.py b/dev-support/git-jira-validation/git_jira_fix_version_check.py
@@ -30,7 +30,7 @@
 
 from jira import JIRA
 
-jira_project_name = input("JIRA Project Name (e.g HADOOP / OZONE etc): ") \
+jira_project_name = input("JIRA Project Name (default: HADOOP): ") \
                     or "HADOOP"
 # Define project_jira_keys with - appended. e.g for HADOOP Jiras,
 # project_jira_keys should include HADOOP-, HDFS-, YARN-, MAPREDUCE-
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -2192,6 +2192,13 @@ function hadoop_daemon_handler
   case ${daemonmode} in
     status)
       hadoop_status_daemon "${daemon_pidfile}"
+      if [[ $? == 0 ]]; then
+        echo "${daemonname} is running as process $(cat "${daemon_pidfile}")."
+      elif [[ $? == 1 ]]; then
+        echo "${daemonname} is stopped."
+      else
+        hadoop_error "hadoop_status_daemon error."
+      fi
       exit $?
     ;;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java
@@ -874,7 +874,7 @@ private BigInteger getNameserviceAggregatedBigInt(
 
   /**
    * Fetches the most active namenode memberships for all known nameservices.
-   * The fetched membership may not or may not be active. Excludes expired
+   * The fetched membership may or may not be active. Excludes expired
    * memberships.
    * @throws IOException if the query could not be performed.
    * @return List of the most active NNs from each known nameservice.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java
@@ -170,7 +170,7 @@ private void chooseEvenlyFromRemainingRacks(Node writer,
     NotEnoughReplicasException lastException = e;
     int bestEffortMaxNodesPerRack = maxNodesPerRack;
     while (results.size() != totalReplicaExpected &&
-        numResultsOflastChoose != results.size()) {
+        bestEffortMaxNodesPerRack < totalReplicaExpected) {
       // Exclude the chosen nodes
       final Set<Node> newExcludeNodes = new HashSet<>();
       for (DatanodeStorageInfo resultStorage : results) {
@@ -192,11 +192,22 @@ private void chooseEvenlyFromRemainingRacks(Node writer,
       } finally {
         excludedNodes.addAll(newExcludeNodes);
       }
+      // To improve performance, the maximum value of 'bestEffortMaxNodesPerRack'
+      // is calculated only when it is not possible to select a node.
+      if (numResultsOflastChoose == results.size()) {
+        Map<String, Integer> nodesPerRack = new HashMap<>();
+        for (DatanodeStorageInfo dsInfo : results) {
+          String rackName = dsInfo.getDatanodeDescriptor().getNetworkLocation();
+          nodesPerRack.merge(rackName, 1, Integer::sum);
+        }
+        bestEffortMaxNodesPerRack =
+            Math.max(bestEffortMaxNodesPerRack, Collections.max(nodesPerRack.values()));
+      }
     }
 
-    if (numResultsOflastChoose != totalReplicaExpected) {
+    if (results.size() != totalReplicaExpected) {
       LOG.debug("Best effort placement failed: expecting {} replicas, only "
-          + "chose {}.", totalReplicaExpected, numResultsOflastChoose);
+          + "chose {}.", totalReplicaExpected, results.size());
       throw lastException;
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -186,8 +186,8 @@ public void sendOOB() throws IOException, InterruptedException {
     if (br == null) {
       return;
     }
-    // This doesn't need to be in a critical section. Althogh the client
-    // can resue the connection to issue a different request, trying sending
+    // This doesn't need to be in a critical section. Although the client
+    // can reuse the connection to issue a different request, trying sending
     // an OOB through the recently closed block receiver is harmless.
     LOG.info("Sending OOB to peer: {}", peer);
     br.sendOOB();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -624,7 +624,7 @@ private static FileState analyzeFileState(
       //    timeout, or because of an HA failover. In that case, we know
       //    by the fact that the client is re-issuing the RPC that it
       //    never began to write to the old block. Hence it is safe to
-      //    to return the existing block.
+      //    return the existing block.
       // 3) This is an entirely bogus request/bug -- we should error out
       //    rather than potentially appending a new block with an empty
       //    one in the middle, etc
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
@@ -327,7 +327,7 @@ public void testPipelineRecoveryOnOOB() throws Exception {
       // Wait long enough to receive an OOB ack before closing the file.
       GenericTestUtils.waitForThreadTermination(
           "Async datanode shutdown thread", 100, 10000);
-      // Retart the datanode 
+      // Restart the datanode
       cluster.restartDataNode(0, true);
       // The following forces a data packet and end of block packets to be sent. 
       out.close();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -52,6 +53,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Arrays;
 import java.util.BitSet;
 import java.util.Iterator;
 import java.util.List;
@@ -515,4 +517,63 @@ public void testReconstrutionWithBusyBlock1() throws Exception {
     assertEquals(9, bm.countNodes(blockInfo).liveReplicas());
   }
 
+  @Test
+  public void testReconstructionWithStorageTypeNotEnough() throws Exception {
+    final HdfsConfiguration conf = new HdfsConfiguration();
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+
+    // Nine disk node eleven archive node.
+    int numDn = groupSize * 2 + 2;
+    StorageType[][] storageTypes = new StorageType[numDn][];
+    Arrays.fill(storageTypes, 0, groupSize,
+        new StorageType[]{StorageType.DISK, StorageType.DISK});
+    Arrays.fill(storageTypes, groupSize, numDn,
+        new StorageType[]{StorageType.ARCHIVE, StorageType.ARCHIVE});
+
+    // Nine disk racks and one archive rack.
+    String[] racks = {
+        "/rack1", "/rack2", "/rack3", "/rack4", "/rack5", "/rack6", "/rack7", "/rack8",
+        "/rack9", "/rack0", "/rack0", "/rack0", "/rack0", "/rack0", "/rack0", "/rack0",
+        "/rack0", "/rack0", "/rack0", "/rack0"};
+
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDn)
+        .storageTypes(storageTypes)
+        .racks(racks)
+        .build();
+    cluster.waitActive();
+    DistributedFileSystem fs = cluster.getFileSystem();
+    fs.enableErasureCodingPolicy(
+        StripedFileTestUtil.getDefaultECPolicy().getName());
+
+    try {
+      fs.mkdirs(dirPath);
+      fs.setStoragePolicy(dirPath, "COLD");
+      fs.setErasureCodingPolicy(dirPath,
+          StripedFileTestUtil.getDefaultECPolicy().getName());
+      DFSTestUtil.createFile(fs, filePath,
+          cellSize * dataBlocks * 2, (short) 1, 0L);
+
+      // Stop one dn.
+      LocatedBlocks blks = fs.getClient().getLocatedBlocks(filePath.toString(), 0);
+      LocatedStripedBlock block = (LocatedStripedBlock) blks.getLastLocatedBlock();
+      DatanodeInfo dnToStop = block.getLocations()[0];
+      cluster.stopDataNode(dnToStop.getXferAddr());
+      cluster.setDataNodeDead(dnToStop);
+
+      // Wait for reconstruction to happen.
+      StripedFileTestUtil.waitForReconstructionFinished(filePath, fs, groupSize);
+      blks = fs.getClient().getLocatedBlocks(filePath.toString(), 0);
+      block = (LocatedStripedBlock) blks.getLastLocatedBlock();
+      BitSet bitSet = new BitSet(groupSize);
+      for (byte index : block.getBlockIndices()) {
+        bitSet.set(index);
+      }
+      for (int i = 0; i < groupSize; i++) {
+        Assert.assertTrue(bitSet.get(i));
+      }
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
 }
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
@@ -139,7 +139,7 @@
     <jna.version>5.2.0</jna.version>
     <gson.version>2.9.0</gson.version>
     <metrics.version>3.2.4</metrics.version>
-    <netty4.version>4.1.89.Final</netty4.version>
+    <netty4.version>4.1.94.Final</netty4.version>
     <snappy-java.version>1.1.10.1</snappy-java.version>
     <lz4-java.version>1.7.1</lz4-java.version>
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -4380,6 +4380,16 @@ public static boolean isAclEnabled(Configuration conf) {
       FEDERATION_GPG_PREFIX + "subcluster.heartbeat.expiration-ms";
   public static final long DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS = TimeUnit.MINUTES.toMillis(30);
 
+  /** Keytab for GPG. **/
+  public static final String GPG_KEYTAB = FEDERATION_GPG_PREFIX + "keytab.file";
+
+  /** The Kerberos principal for the globalpolicygenerator.*/
+  public static final String GPG_PRINCIPAL = FEDERATION_GPG_PREFIX + "kerberos.principal";
+
+  /** The Kerberos principal hostname for the yarn gpg.*/
+  public static final String GPG_KERBEROS_PRINCIPAL_HOSTNAME_KEY = FEDERATION_GPG_PREFIX +
+      "kerberos.principal.hostname";
+
   /**
    * Connection and Read timeout from the Router to RM.
    */
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -5394,4 +5394,36 @@
     <value>100ms</value>
   </property>
 
+  <property>
+    <name>yarn.federation.gpg.keytab.file</name>
+    <value></value>
+    <description>
+      The keytab file used by gpg to login as its
+      service principal. The principal name is configured with
+      dfs.federation.router.kerberos.principal.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.federation.gpg.kerberos.principal</name>
+    <value></value>
+    <description>
+      The GPG service principal. This is typically set to
+      gpg/_HOST@REALM.TLD. Each GPG will substitute _HOST with its
+      own fully qualified hostname at startup. The _HOST placeholder
+      allows using the same configuration setting on both GPG setup.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.federation.gpg.kerberos.principal.hostname</name>
+    <value></value>
+    <description>
+      Optional.
+      The hostname for the Router containing this
+      configuration file. Will be different for each machine.
+      Defaults to current hostname.
+    </description>
+  </property>
+
 </configuration>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
@@ -86,6 +86,19 @@
       <classifier>jdk8</classifier>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minikdc</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-auth</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+
   </dependencies>
 
   <build>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/secure/AbstractGlobalPolicyGeneratorTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/secure/AbstractGlobalPolicyGeneratorTest.java
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/secure/TestGpgSecureLogins.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/secure/TestGpgSecureLogins.java
