From 57e62ae07f1c4eb8adccd9c61fc909080ca76c53 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 5 Sep 2024 14:35:50 +0100 Subject: [PATCH] Revert "YARN-11664. Remove HDFS Binaries/Jars Dependency From Yarn (#6631)" This reverts commit 6c01490f14b65f43196e1f235c51749a712e7338. --- .../apache/hadoop/fs/HdfsCommonConstants.java | 47 ------------------- .../protocol/datatransfer/package-info.java | 25 ---------- .../protocol/datatransfer/IOStreamPair.java | 6 +-- .../delegation/DelegationTokenIdentifier.java | 9 +--- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 +-- .../hadoop/yarn/service/ServiceMaster.java | 4 +- .../yarn/service/client/ServiceClient.java | 6 +-- .../yarn/service/client/TestServiceCLI.java | 4 +- .../logaggregation/AggregatedLogFormat.java | 6 +-- .../tfile/LogAggregationTFileController.java | 4 +- 10 files changed, 18 insertions(+), 100 deletions(-) delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HdfsCommonConstants.java delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java rename {hadoop-common-project/hadoop-common => hadoop-hdfs-project/hadoop-hdfs-client}/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/IOStreamPair.java (85%) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HdfsCommonConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HdfsCommonConstants.java deleted file mode 100644 index f6c3ca4517d55..0000000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HdfsCommonConstants.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.io.Text; - -/** - * This class contains constants for configuration keys and default values. - */ -@InterfaceAudience.LimitedPrivate({"YARN", "HDFS"}) -@InterfaceStability.Evolving -public final class HdfsCommonConstants { - - /** - * Represents the kind of delegation token used for HDFS. - * This is a constant string value "HDFS_DELEGATION_TOKEN". - */ - public static final Text HDFS_DELEGATION_KIND = - new Text("HDFS_DELEGATION_TOKEN"); - - /** - * DFS_ADMIN configuration: {@value}. - */ - public static final String DFS_ADMIN = "dfs.cluster.administrators"; - - private HdfsCommonConstants() { - } - -} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java deleted file mode 100644 index d2b8638b96e20..0000000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * This package provides access to following class. - * {@link org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair} class. - */ -@InterfaceAudience.Private -package org.apache.hadoop.hdfs.protocol.datatransfer; - -import org.apache.hadoop.classification.InterfaceAudience; \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/IOStreamPair.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/IOStreamPair.java similarity index 85% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/IOStreamPair.java rename to hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/IOStreamPair.java index 597c804d6fc51..4ec73e3378445 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/IOStreamPair.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/IOStreamPair.java @@ -22,13 +22,13 @@ import java.io.InputStream; import java.io.OutputStream; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.io.IOUtils; /** - * This class wraps an InputStream and an OutputStream. - * Both the InputStream and OutputStream are closed on close call. - * This class is moved from HDFS module to COMMON module for removing HDFS dependencies from YARN. + * A little struct class to wrap an InputStream and an OutputStream. */ +@InterfaceAudience.Private public class IOStreamPair implements Closeable { public final InputStream in; public final OutputStream out; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java index fa90342d419e8..1f4c36f679670 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java @@ -26,7 +26,6 @@ import org.apache.commons.collections.map.LRUMap; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.fs.HdfsCommonConstants; import org.apache.hadoop.hdfs.web.WebHdfsConstants; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; @@ -42,12 +41,8 @@ @InterfaceAudience.Private public class DelegationTokenIdentifier extends AbstractDelegationTokenIdentifier { - - /** - * The value is referenced from {@link HdfsCommonConstants#HDFS_DELEGATION_KIND}. - */ - @Deprecated - public static final Text HDFS_DELEGATION_KIND = HdfsCommonConstants.HDFS_DELEGATION_KIND; + public static final Text HDFS_DELEGATION_KIND = + new Text("HDFS_DELEGATION_TOKEN"); @SuppressWarnings("unchecked") private static Map ugiCache = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 601d9c772e795..dd3193fdadff2 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -20,7 +20,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.fs.HdfsCommonConstants; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -382,11 +381,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_XATTRS_ENABLED_KEY = "dfs.namenode.xattrs.enabled"; public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true; - /** - * The value is referenced from {@link HdfsCommonConstants#DFS_ADMIN}. - */ - @Deprecated - public static final String DFS_ADMIN = HdfsCommonConstants.DFS_ADMIN; + public static final String DFS_ADMIN = "dfs.cluster.administrators"; public static final String DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY = "dfs.https.server.keystore.resource"; public static final String DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT = "ssl-server.xml"; public static final String DFS_SERVER_HTTPS_KEYPASSWORD_KEY = "ssl.server.keystore.keypassword"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java index 97ed7ec2a4836..00883d94eefe6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java @@ -23,8 +23,8 @@ import org.apache.commons.cli.Options; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.HdfsCommonConstants; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; @@ -230,7 +230,7 @@ private static void removeHdfsDelegationToken(UserGroupInformation user) { while (iter.hasNext()) { Token token = iter.next(); if (token.getKind().equals( - HdfsCommonConstants.HDFS_DELEGATION_KIND)) { + DelegationTokenIdentifier.HDFS_DELEGATION_KIND)) { LOG.info("Remove HDFS delegation token {}.", token); iter.remove(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 2fda45c9b490e..9098bb4a751fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -31,10 +31,10 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.HdfsCommonConstants; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.registry.client.api.RegistryConstants; @@ -1707,12 +1707,12 @@ private boolean checkPermissions(Path dependencyLibTarGzip) throws YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); AccessControlList dfsAdminAcl = new AccessControlList( - getConfig().get(HdfsCommonConstants.DFS_ADMIN, " ")); + getConfig().get(DFSConfigKeys.DFS_ADMIN, " ")); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); if (!yarnAdminAcl.isUserAllowed(ugi) && !dfsAdminAcl.isUserAllowed(ugi)) { LOG.error("User must be on the {} or {} list to have permission to " + "upload AM dependency tarball", YarnConfiguration.YARN_ADMIN_ACL, - HdfsCommonConstants.DFS_ADMIN); + DFSConfigKeys.DFS_ADMIN); return false; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java index 3b5f29a7ca1a3..f75c0afdbebb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java @@ -20,9 +20,9 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.HdfsCommonConstants; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.client.cli.ApplicationCLI; @@ -138,7 +138,7 @@ public void setup() throws Throwable { basedir.mkdirs(); } yarnAdminNoneAclProp = YarnConfiguration.YARN_ADMIN_ACL + "=none"; - dfsAdminAclProp = HdfsCommonConstants.DFS_ADMIN + "=" + + dfsAdminAclProp = DFSConfigKeys.DFS_ADMIN + "=" + UserGroupInformation.getCurrentUser(); System.setProperty(YarnServiceConstants.PROPERTY_LIB_DIR, basedir .getAbsolutePath()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 1a35a6042561b..5a752064bce1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -52,7 +52,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.ClusterStorageCapacityExceededException; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -61,6 +60,7 @@ import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SecureIOUtils; import org.apache.hadoop.io.Writable; @@ -547,7 +547,7 @@ public void append(LogKey logKey, LogValue logValue) throws IOException { } @Override - public void close() throws ClusterStorageCapacityExceededException { + public void close() throws DSQuotaExceededException { try { if (writer != null) { writer.close(); @@ -557,7 +557,7 @@ public void close() throws ClusterStorageCapacityExceededException { } finally { try { this.fsDataOStream.close(); - } catch (ClusterStorageCapacityExceededException e) { + } catch (DSQuotaExceededException e) { LOG.error("Exception in closing {}", this.fsDataOStream.getClass(), e); throw e; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java index ccfd8cef6c592..4b282488c6d56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java @@ -38,12 +38,12 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.ClusterStorageCapacityExceededException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.HarFs; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; @@ -99,7 +99,7 @@ public void closeWriter() throws LogAggregationDFSException { if (this.writer != null) { try { this.writer.close(); - } catch (ClusterStorageCapacityExceededException e) { + } catch (DSQuotaExceededException e) { throw new LogAggregationDFSException(e); } finally { this.writer = null;